iHore/vLLM
8
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[Mini-VLLM] Remove LoRA, plugins, speech-to-text and scale-out surfaces

Browse Source 
Production runs one static model behind the OpenAI-compatible API. Every
adapter path and non-OpenAI HTTP surface is unreachable there.

- Delete the LoRA stack: vllm/lora, LoRARequest on the engine wire, the
  scheduler, worker, cudagraph and MoE plumbing, LoRA metrics, the lora
  CLI group, and the LoRA tests.
- Drop the plugin loader in vllm/plugins. Platform resolution is now
  CUDA-only and the general/stat-logger plugin hooks are gone.
- Delete the speech-to-text and realtime entrypoints, the scale-out
  render routes, and vllm/config/speech_to_text.py. GenerationTask is
  generate-only.
- Cut DecodeBenchConnector and the disagg-prefill doc and image. The
  Multi, Offloading and Example connectors stay; production uses the
  CPU KV offload tier.
- Delete the ROCm-only fused KDA chunk kernel (2346 lines) with its CMake
  arch gate, declarations and op registrations. The CUDA build never
  defined the guard and no Python caller existed.
- Delete the orphan Mistral and Pixtral chat fixtures and examples/.
- Fix _check_model: removing the LoRA early return made every request
  fail with "The model ... does not exist. Valid aliases: ...".

Verified on an RTX 3090: import sweep reports 1242 modules with 7 known
optional-dependency failures; a live server answers chat, completions,
responses, tokenize and detokenize with zero log errors and 404 for the
removed surfaces; collection yields 8558 tests with the 3 pre-existing
errors; ruff adds no new finding; the incremental _C build succeeds.

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
MOVZX
2026-10-19 21:06:19 +07:00
co-authored by Hinata
parent 5f3c7bd8ab
commit 5aa2ac64f8
269 changed files with 417 additions and 29932 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CMakeLists.txt
+1 -15
View File
@@ -1093,8 +1093,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
# (gfx942 / gfx950), so it is only emitted when one of those is in the arch
# list. A multi-arch build that includes them still emits this source for
# every arch in that list. The runtime gate in kda_decode.py keeps the kernel
# off unsupported devices. Fused prefill KDA kernels is Only measured on
# gfx950, so it is only built when gfx950 is in the arch list.
# off unsupported devices.
if(VLLM_GPU_LANG STREQUAL "HIP")
set(FUSED_KDA_DECODE_HIP_ARCHS ${VLLM_GPU_ARCHES})
list(FILTER FUSED_KDA_DECODE_HIP_ARCHS INCLUDE REGEX "gfx942|gfx950")
@@ -1106,15 +1105,6 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
"Building fused KDA decode for HIP archs: ${FUSED_KDA_DECODE_HIP_ARCHS}")
endif()
set(FUSED_KDA_CHUNK_HIP_ARCHS ${VLLM_GPU_ARCHES})
list(FILTER FUSED_KDA_CHUNK_HIP_ARCHS INCLUDE REGEX "gfx950")
if(FUSED_KDA_CHUNK_HIP_ARCHS)
set(FUSED_KDA_CHUNK_HIP_SRC
"csrc/kimi_k3/fused_kda_chunk_kernel_rocm.cu")
list(APPEND VLLM_STABLE_EXT_SRC "${FUSED_KDA_CHUNK_HIP_SRC}")
message(STATUS
"Building fused KDA chunk for HIP archs: ${FUSED_KDA_CHUNK_HIP_ARCHS}")
endif()
endif()
message(STATUS "Enabling C_stable extension.")
@@ -1165,10 +1155,6 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
target_compile_definitions(_C_stable_libtorch PRIVATE
VLLM_ENABLE_FUSED_KDA_DECODE=1)
endif()
if(FUSED_KDA_CHUNK_HIP_SRC)
target_compile_definitions(_C_stable_libtorch PRIVATE
VLLM_ENABLE_FUSED_KDA_CHUNK=1)
endif()
endif()
# On ROCm, _C_stable_libtorch calls raw HIP APIs (e.g. hipGetDevice in
csrc/kimi_k3/fused_kda_chunk_kernel_rocm.cu
-2346
View File
File diff suppressed because it is too large Load Diff
csrc/ops.h
-32
View File
@@ -281,38 +281,6 @@ void fused_gdn_decode_post_conv_mtp(
#endif
#ifdef VLLM_ENABLE_FUSED_KDA_CHUNK
void fused_kda_prologue(
torch::stable::Tensor const& q, torch::stable::Tensor const& k,
torch::stable::Tensor const& v, torch::stable::Tensor const& raw_g,
torch::stable::Tensor const& raw_beta, torch::stable::Tensor const& a_log,
torch::stable::Tensor const& dt_bias, torch::stable::Tensor& qg,
torch::stable::Tensor& w, torch::stable::Tensor& u,
torch::stable::Tensor& kg_t, torch::stable::Tensor& aqk,
torch::stable::Tensor& decay, torch::stable::Tensor const& cu_seqlens,
torch::stable::Tensor const& chunk_indices,
std::optional<torch::stable::Tensor> conv_weight,
std::optional<torch::stable::Tensor> conv_state,
std::optional<torch::stable::Tensor> conv_state_indices,
std::optional<torch::stable::Tensor> conv_has_initial_state, double scale,
double lower_bound);
void fused_kda_chunk(
torch::stable::Tensor const& qg, torch::stable::Tensor const& w,
torch::stable::Tensor const& u, torch::stable::Tensor const& kg_t,
torch::stable::Tensor const& aqk, torch::stable::Tensor const& decay,
std::optional<torch::stable::Tensor> initial_state,
std::optional<torch::stable::Tensor> final_state,
torch::stable::Tensor& out, torch::stable::Tensor const& cu_seqlens,
torch::stable::Tensor const& chunk_offsets, double scale,
std::optional<torch::stable::Tensor> group_state, int64_t groups,
std::optional<torch::stable::Tensor> checkpoint_state,
std::optional<torch::stable::Tensor> checkpoint_offsets,
std::optional<torch::stable::Tensor> checkpoint_state_indices,
std::optional<torch::stable::Tensor> state_cache,
std::optional<torch::stable::Tensor> state_indices,
std::optional<torch::stable::Tensor> has_initial_state);
#endif
// Sampler kernels (shared CUDA/ROCm)
void apply_repetition_penalties_(
csrc/torch_bindings.cpp
-23
View File
@@ -424,25 +424,6 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
"str output_gate_activation='silu') -> ()");
#endif
#ifdef VLLM_ENABLE_FUSED_KDA_CHUNK
ops.def(
"fused_kda_prologue("
"Tensor q, Tensor k, Tensor v, Tensor raw_g, Tensor raw_beta, "
"Tensor A_log, Tensor dt_bias, Tensor! qg, Tensor! w, Tensor! u, "
"Tensor! kg_t, Tensor! aqk, Tensor! decay, Tensor cu_seqlens, "
"Tensor chunk_indices, Tensor? conv_weight, Tensor(e!)? conv_state, "
"Tensor? conv_state_indices, Tensor? conv_has_initial_state, "
"float scale, float lower_bound) -> ()");
ops.def(
"fused_kda_chunk("
"Tensor qg, Tensor w, Tensor u, Tensor kg_t, Tensor aqk, Tensor decay, "
"Tensor? initial_state, Tensor(a!)? final_state, Tensor! out, "
"Tensor cu_seqlens, Tensor chunk_offsets, float scale, "
"Tensor(b!)? group_state, int groups, "
"Tensor(c!)? checkpoint_state=None, Tensor? checkpoint_offsets=None, "
"Tensor? checkpoint_state_indices=None, Tensor(d!)? state_cache=None, "
"Tensor? state_indices=None, Tensor? has_initial_state=None) -> ()");
#endif
// Apply repetition penalties to logits in-place.
ops.def(
@@ -684,10 +665,6 @@ STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, ops) {
TORCH_BOX(&fused_gdn_decode_post_conv_mtp));
#endif
#ifdef VLLM_ENABLE_FUSED_KDA_CHUNK
ops.impl("fused_kda_prologue", TORCH_BOX(&fused_kda_prologue));
ops.impl("fused_kda_chunk", TORCH_BOX(&fused_kda_chunk));
#endif
// Sampler kernels (shared CUDA/ROCm)
ops.impl("apply_repetition_penalties_",
examples/template_chatml.jinja
-2
View File
@@ -1,2 +0,0 @@
{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content']}}{% if (loop.last and add_generation_prompt) or not loop.last %}{{ '<|im_end|>' + '\n'}}{% endif %}{% endfor %}
{% if add_generation_prompt and messages[-1]['role'] != 'assistant' %}{{ '<|im_start|>assistant\n' }}{% endif %}
pyproject.toml
-4
View File
@@ -41,10 +41,6 @@ Slack="https://slack.vllm.ai/"
[project.scripts]
vllm = "vllm.entrypoints.cli.main:main"
[project.entry-points."vllm.general_plugins"]
lora_filesystem_resolver = "vllm.plugins.lora_resolvers.filesystem_resolver:register_filesystem_resolver"
lora_hf_hub_resolver = "vllm.plugins.lora_resolvers.hf_hub_resolver:register_hf_hub_resolver"
[tool.setuptools_scm]
# no extra settings needed, presence enables setuptools-scm
tests/conftest.py
-19
View File
@@ -1534,29 +1534,10 @@ def num_gpus_available():
temp_dir = tempfile.gettempdir()
_dummy_opt_path = os.path.join(temp_dir, "dummy_opt")
_dummy_llava_path = os.path.join(temp_dir, "dummy_llava")
_dummy_gemma2_embedding_path = os.path.join(temp_dir, "dummy_gemma2_embedding")
@pytest.fixture
def dummy_opt_path():
json_path = os.path.join(_dummy_opt_path, "config.json")
if not os.path.exists(_dummy_opt_path):
hf_api().snapshot_download(
repo_id="facebook/opt-125m",
local_dir=_dummy_opt_path,
ignore_patterns=["*.bin", "*.bin.index.json", "*.pt", "*.h5", "*.msgpack"],
)
assert os.path.exists(json_path)
with open(json_path) as f:
config = json.load(f)
config["architectures"] = ["MyOPTForCausalLM"]
with open(json_path, "w") as f:
json.dump(config, f)
return _dummy_opt_path
@pytest.fixture
def dummy_llava_path():
json_path = os.path.join(_dummy_llava_path, "config.json")
tests/detokenizer/test_stop_string_while_stop_model_terminates.py
-1
View File
@@ -39,7 +39,6 @@ def _make_request(stop, include_stop_str_in_output: bool, min_tokens: int = 0):
sampling_params=params,
pooling_params=None,
arrival_time=0.0,
lora_request=None,
cache_salt=None,
data_parallel_rank=None,
)
tests/distributed/test_distributed_oot.py
-28
View File
@@ -1,28 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import pytest
from tests.plugins_tests.test_oot_registration_online import (
run_and_test_dummy_opt_api_server,
)
def test_distributed_oot(dummy_opt_path: str):
run_and_test_dummy_opt_api_server(dummy_opt_path, tp=2)
@pytest.mark.parametrize("enforce_eager", [True, False])
def test_distributed_oot_pp_raw_input_tokens(dummy_opt_path: str, enforce_eager: bool):
# The model asserts that input_ids reaches every pipeline stage.
overrides = {"architectures": ["MyOPTRawInputTokensForCausalLM"]}
extra_args = ["-pp", "2", "--hf-overrides", json.dumps(overrides)]
if enforce_eager:
extra_args.append("--enforce-eager")
run_and_test_dummy_opt_api_server(
dummy_opt_path,
extra_args=extra_args,
env_dict={"VLLM_USE_V2_MODEL_RUNNER": "1"},
)
tests/distributed/test_kv_cache_events.py
-10
View File
@@ -24,9 +24,7 @@ class _LegacyBlockStored(
parent_block_hash: bytes | None
token_ids: list[int]
block_size: int
lora_id: int | None
medium: str | None
lora_name: str | None
extra_keys: list[tuple[Any, ...] | None] | None = None
group_idx: int | None = None
kv_cache_spec_kind: str | None = None
@@ -58,9 +56,7 @@ class _PreSessionBlockStored(
parent_block_hash: bytes | None
token_ids: list[int]
block_size: int
lora_id: int | None
medium: str | None
lora_name: str | None
extra_keys: list[tuple[Any, ...] | None] | None = None
group_idx: int | None = None
kv_cache_spec_kind: str | None = None
@@ -79,9 +75,7 @@ def _make_block_stored(
parent_block_hash=None,
token_ids=[1, 2, 3, 4],
block_size=4,
lora_id=None,
medium="GPU",
lora_name=None,
group_idx=group_idx,
kv_cache_spec_sliding_window=kv_cache_spec_sliding_window,
locality=locality,
@@ -183,9 +177,7 @@ def test_block_stored_locality_is_wire_compatible():
parent_block_hash=None,
token_ids=[1, 2, 3, 4],
block_size=4,
lora_id=None,
medium="GPU",
lora_name=None,
group_idx=2,
kv_cache_spec_sliding_window=128,
)
@@ -221,9 +213,7 @@ def test_block_stored_session_id_is_wire_compatible():
parent_block_hash=None,
token_ids=[1, 2, 3, 4],
block_size=4,
lora_id=None,
medium="GPU",
lora_name=None,
group_idx=2,
kv_cache_spec_sliding_window=128,
locality="LOCAL",
tests/entrypoints/conftest.py
-34
View File
@@ -65,37 +65,3 @@ def sample_structured_outputs_choices():
"Swift",
"Kotlin",
]
@pytest.fixture(scope="session")
def qwen3_lora_files():
"""Download Qwen3 LoRA files once per test session."""
from vllm.transformers_utils.repo_utils import hf_api
return hf_api().snapshot_download(repo_id="charent/self_cognition_Alice")
@pytest.fixture(scope="session")
def qwen3_meowing_lora_files():
"""Download Qwen3 LoRA files once per test session."""
from vllm.transformers_utils.repo_utils import hf_api
return hf_api().snapshot_download(repo_id="Jackmin108/Qwen3-0.6B-Meow-LoRA")
@pytest.fixture(scope="session")
def qwen3_woofing_lora_files():
"""Download Qwen3 LoRA files once per test session."""
from vllm.transformers_utils.repo_utils import hf_api
return hf_api().snapshot_download(repo_id="Jackmin108/Qwen3-0.6B-Woof-LoRA")
@pytest.fixture(scope="session")
def opt125_lora_files() -> str:
"""Download opt-125m LoRA files once per test session."""
from vllm.transformers_utils.repo_utils import hf_api
return hf_api().snapshot_download(
repo_id="peft-internal-testing/opt-125m-dummy-lora"
)
tests/entrypoints/launchers/test_cli_args.py
+4 -124
View File
@@ -1,28 +1,16 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import pytest
from tests.utils import VLLM_PATH
from vllm.engine.arg_utils import AsyncEngineArgs
from vllm.entrypoints.launchers.cli_args import (
make_arg_parser,
validate_parsed_serve_args,
)
from vllm.entrypoints.openai.models.protocol import LoRAModulePath
from vllm.exceptions import VLLMValidationError
from vllm.utils.argparse_utils import FlexibleArgumentParser
LORA_MODULE = {
"name": "module2",
"path": "/path/to/module2",
"base_model_name": "llama",
}
CHATML_JINJA_PATH = VLLM_PATH / "examples/template_chatml.jinja"
assert CHATML_JINJA_PATH.exists()
def _build_vllm_parsers():
vllm_parser = FlexibleArgumentParser()
@@ -96,87 +84,6 @@ def test_logging_config_cli_args(serve_parser):
assert config.pylogging_config_file == "/tmp/flat.json"
### Tests for LoRA module parsing
def test_valid_key_value_format(serve_parser):
# Test old format: name=path
args = serve_parser.parse_args(
[
"--lora-modules",
"module1=/path/to/module1",
]
)
expected = [LoRAModulePath(name="module1", path="/path/to/module1")]
assert args.lora_modules == expected
def test_valid_json_format(serve_parser):
# Test valid JSON format input
args = serve_parser.parse_args(
[
"--lora-modules",
json.dumps(LORA_MODULE),
]
)
expected = [
LoRAModulePath(name="module2", path="/path/to/module2", base_model_name="llama")
]
assert args.lora_modules == expected
def test_invalid_json_format(serve_parser):
# Test invalid JSON format input, missing closing brace
with pytest.raises(SystemExit):
serve_parser.parse_args(
["--lora-modules", '{"name": "module3", "path": "/path/to/module3"']
)
def test_invalid_type_error(serve_parser):
# Test type error when values are not JSON or key=value
with pytest.raises(SystemExit):
serve_parser.parse_args(
[
"--lora-modules",
"invalid_format", # This is not JSON or key=value format
]
)
def test_invalid_json_field(serve_parser):
# Test valid JSON format but missing required fields
with pytest.raises(SystemExit):
serve_parser.parse_args(
[
"--lora-modules",
'{"name": "module4"}', # Missing required 'path' field
]
)
def test_empty_values(serve_parser):
# Test when no LoRA modules are provided
args = serve_parser.parse_args(["--lora-modules", ""])
assert args.lora_modules == []
def test_multiple_valid_inputs(serve_parser):
# Test multiple valid inputs (both old and JSON format)
args = serve_parser.parse_args(
[
"--lora-modules",
"module1=/path/to/module1",
json.dumps(LORA_MODULE),
]
)
expected = [
LoRAModulePath(name="module1", path="/path/to/module1"),
LoRAModulePath(
name="module2", path="/path/to/module2", base_model_name="llama"
),
]
assert args.lora_modules == expected
### Tests for serve argument validation that run prior to loading
@@ -241,10 +148,12 @@ def test_reasoning_parser_none_disables_parser(serve_parser, value, expected):
assert args.reasoning_parser == expected
def test_chat_template_validation_for_happy_paths(serve_parser):
def test_chat_template_validation_for_happy_paths(serve_parser, tmp_path):
"""Ensure validation passes if the chat template exists."""
chat_template = tmp_path / "template.jinja"
chat_template.write_text("{{ messages }}")
args = serve_parser.parse_args(
args=["--chat-template", CHATML_JINJA_PATH.absolute().as_posix()]
args=["--chat-template", chat_template.absolute().as_posix()]
)
validate_parsed_serve_args(args)
@@ -389,32 +298,3 @@ def test_served_model_name_parsing(tmp_path, vllm_parser, args, raises):
else:
with pytest.raises(raises):
vllm_parser.parse_args(args=args)
### Tests for LoRA target modules parsing
def test_lora_target_modules_single(serve_parser):
"""Test parsing single lora-target-modules argument."""
args = serve_parser.parse_args(
args=["--enable-lora", "--lora-target-modules", "o_proj"]
)
assert args.lora_target_modules == ["o_proj"]
def test_lora_target_modules_multiple(serve_parser):
"""Test parsing multiple lora-target-modules arguments."""
args = serve_parser.parse_args(
args=[
"--enable-lora",
"--lora-target-modules",
"o_proj",
"qkv_proj",
"down_proj",
]
)
assert args.lora_target_modules == ["o_proj", "qkv_proj", "down_proj"]
def test_lora_target_modules_default_none(serve_parser):
"""Test that lora-target-modules defaults to None."""
args = serve_parser.parse_args(args=[])
assert args.lora_target_modules is None
tests/entrypoints/multimodal/openai/chat_completion/test_default_mm_loras.py
-98
View File
@@ -1,98 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
import openai # use the official client for correctness check
import pytest
import pytest_asyncio
from tests.conftest import AudioTestAssets
from tests.utils import RemoteOpenAIServer
from vllm.transformers_utils.repo_utils import hf_api
# NOTE - the tests in this module are currently analogous to test_chat, but are
# separated to avoid OOM killing due to module-scoped servers, since we
# need a multimodal model for these tests.
# Contains a modality specific lora alongside the base model
MULTIMODAL_MODEL_NAME = hf_api().snapshot_download(
"microsoft/Phi-4-multimodal-instruct"
)
AUDIO_LORA_PATH = os.path.join(MULTIMODAL_MODEL_NAME, "speech-lora")
ACTIVE_MM_LORA_RESPONSE = "Spoken text: The first words I spoke in the original chronograph, a little piece of practical poetry. Mary had a little lamb, it slept with quite a snow, and everywhere that Mary went, the lamb was sure to go." # noqa: E501
@pytest.fixture(scope="module")
def multimodal_server():
args = [
# use half precision for speed and memory savings in CI environment
"--dtype",
"half",
"--max-model-len",
"4096",
"--enforce-eager",
# lora config below
"--enable-lora",
"--lora-modules",
f"speech={AUDIO_LORA_PATH}",
"--max-lora-rank",
"320",
"--max-num-seqs",
"2",
"--trust-remote-code",
"--gpu-memory-utilization",
"0.8",
"--default-mm-loras",
f'{{"audio": "{AUDIO_LORA_PATH}"}}',
]
with RemoteOpenAIServer(
MULTIMODAL_MODEL_NAME, args, max_wait_seconds=480
) as remote_server:
yield remote_server
@pytest_asyncio.fixture
async def multi_modal_client(multimodal_server):
async with multimodal_server.get_async_client() as async_client:
yield async_client
@pytest.mark.asyncio
@pytest.mark.parametrize(
# base model with default lora should give the same response as lora model
"model_name",
[MULTIMODAL_MODEL_NAME, "speech"],
)
async def test_default_mm_lora_chat_completions(
model_name: str,
multi_modal_client: openai.AsyncOpenAI,
audio_assets: AudioTestAssets,
):
messages = [
{
"role": "user",
"content": [
{
"type": "text",
"text": "Can you transcribe this audio?",
},
{
"type": "audio_url",
"audio_url": {"url": audio_assets[0].url},
},
],
}
]
chat_completion = await multi_modal_client.chat.completions.create(
model=model_name, messages=messages, max_completion_tokens=128, temperature=0.0
)
assert len(chat_completion.choices) > 0
message = chat_completion.choices[0].message
assert message.content is not None and len(message.content) >= 0
assert message.content == ACTIVE_MM_LORA_RESPONSE
tests/entrypoints/openai/chat_completion/test_chat.py
+8 -27
View File
@@ -27,15 +27,7 @@ MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
@pytest.fixture(scope="module")
def zephyr_lora_files():
"""Download zephyr LoRA files once per test session."""
from vllm.transformers_utils.repo_utils import hf_api
return hf_api().snapshot_download(repo_id="typeof/zephyr-7b-beta-lora")
@pytest.fixture(scope="module")
def server(zephyr_lora_files):
def server():
args = [
# use half precision for speed and memory savings in CI environment
"--dtype",
@@ -43,14 +35,6 @@ def server(zephyr_lora_files):
"--max-model-len",
"8192",
"--enforce-eager",
# lora config below
"--enable-lora",
"--lora-modules",
f"zephyr-lora={zephyr_lora_files}",
"--max-lora-rank",
"64",
"--max-cpu-loras",
"2",
"--max-num-seqs",
"128",
]
@@ -67,9 +51,8 @@ async def client(server):
@pytest.mark.asyncio
@pytest.mark.parametrize(
# first test base model, then test loras
"model_name",
[MODEL_NAME, "zephyr-lora"],
[MODEL_NAME],
)
async def test_no_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
messages = [
@@ -91,9 +74,8 @@ async def test_no_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
@pytest.mark.asyncio
@pytest.mark.parametrize(
# just test 1 lora hereafter
"model_name",
[MODEL_NAME, "zephyr-lora"],
[MODEL_NAME],
)
async def test_zero_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
messages = [
@@ -119,7 +101,7 @@ async def test_zero_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
@pytest.mark.asyncio
@pytest.mark.parametrize(
"model_name",
[MODEL_NAME, "zephyr-lora"],
[MODEL_NAME],
)
async def test_some_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
messages = [
@@ -145,7 +127,7 @@ async def test_some_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
@pytest.mark.asyncio
@pytest.mark.parametrize(
"model_name",
[MODEL_NAME, "zephyr-lora"],
[MODEL_NAME],
)
async def test_too_many_chat_logprobs(client: openai.AsyncOpenAI, model_name: str):
messages = [
@@ -254,7 +236,7 @@ async def test_more_than_one_prompt_logprobs_chat(
@pytest.mark.asyncio
@pytest.mark.parametrize(
"model_name",
[MODEL_NAME, "zephyr-lora"],
[MODEL_NAME],
)
async def test_single_chat_session(client: openai.AsyncOpenAI, model_name: str):
messages = [
@@ -297,9 +279,8 @@ async def test_single_chat_session(client: openai.AsyncOpenAI, model_name: str):
@pytest.mark.asyncio
@pytest.mark.parametrize(
# just test 1 lora hereafter
"model_name",
[MODEL_NAME, "zephyr-lora"],
[MODEL_NAME],
)
async def test_chat_streaming(client: openai.AsyncOpenAI, model_name: str):
messages = [
@@ -345,7 +326,7 @@ async def test_chat_streaming(client: openai.AsyncOpenAI, model_name: str):
@pytest.mark.asyncio
@pytest.mark.parametrize(
"model_name",
["HuggingFaceH4/zephyr-7b-beta", "zephyr-lora"],
["HuggingFaceH4/zephyr-7b-beta"],
)
async def test_chat_completion_stream_options(
client: openai.AsyncOpenAI, model_name: str
tests/entrypoints/openai/chat_completion/test_completion_with_function_calling.py
-1
View File
@@ -9,7 +9,6 @@ import openai # use the official client for correctness check
import pytest
import pytest_asyncio
# downloading lora to test lora requests
from tests.utils import ROCM_EXTRA_ARGS, RemoteOpenAIServer
# any model with a chat template should work here
tests/entrypoints/openai/models/test_models.py
+2 -15
View File
@@ -9,12 +9,10 @@ from tests.utils import RemoteOpenAIServer
# any model with a chat template should work here
MODEL_NAME = "Qwen/Qwen3-0.6B"
# technically this needs Mistral-7B-v0.1 as base, but we're not testing
# generation quality here
@pytest.fixture(scope="module")
def server(qwen3_lora_files):
def server():
args = [
# use half precision for speed and memory savings in CI environment
"--dtype",
@@ -22,14 +20,6 @@ def server(qwen3_lora_files):
"--max-model-len",
"8192",
"--enforce-eager",
# lora config below
"--enable-lora",
"--lora-modules",
f"qwen3-lora={qwen3_lora_files}",
"--max-lora-rank",
"64",
"--max-cpu-loras",
"2",
"--max-num-seqs",
"128",
]
@@ -45,12 +35,9 @@ async def client(server):
@pytest.mark.asyncio
async def test_check_models(client: openai.AsyncOpenAI, qwen3_lora_files):
async def test_check_models(client: openai.AsyncOpenAI):
models = await client.models.list()
models = models.data
served_model = models[0]
lora_models = models[1:]
assert served_model.id == MODEL_NAME
assert served_model.root == MODEL_NAME
assert all(lora_model.root == qwen3_lora_files for lora_model in lora_models)
assert lora_models[0].id == "qwen3-lora"
tests/entrypoints/openai/test_render_token_offsets.py
-78
View File
@@ -1,78 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for the token-offsets request/response protocol wiring:
the request flag flowing into ``TokenizeParams`` and the ``GenerateRequest``
serialization boundary. Plain Pydantic field storage is not retested here.
"""
from unittest.mock import Mock
from vllm.config import ModelConfig
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateRequest
from vllm.sampling_params import SamplingParams
def _model_config() -> Mock:
model_config = Mock(spec=ModelConfig)
model_config.max_model_len = 128
return model_config
def test_completion_flag_forwarded_to_tok_params():
"""build_tok_params must forward return_token_offsets, defaulting to
False (zero behavioral change for existing callers) and coercing JSON
null to False via the bool() guard."""
cfg = _model_config()
default = CompletionRequest(model="m", prompt="hi")
assert default.build_tok_params(cfg).return_token_offsets is False
on = CompletionRequest(model="m", prompt="hi", return_token_offsets=True)
assert on.build_tok_params(cfg).return_token_offsets is True
null = CompletionRequest(model="m", prompt="hi", return_token_offsets=None)
assert null.build_tok_params(cfg).return_token_offsets is False
def test_chat_flag_forwarded_to_tok_params():
"""Chat build_tok_params has its own (max_completion_tokens) branch, so
its return_token_offsets forwarding is verified independently."""
cfg = _model_config()
messages = [{"role": "user", "content": "hi"}]
default = ChatCompletionRequest(model="m", messages=messages)
assert default.build_tok_params(cfg).return_token_offsets is False
on = ChatCompletionRequest(model="m", messages=messages, return_token_offsets=True)
assert on.build_tok_params(cfg).return_token_offsets is True
null = ChatCompletionRequest(
model="m", messages=messages, return_token_offsets=None
)
assert null.build_tok_params(cfg).return_token_offsets is False
def test_generate_request_token_offsets_default_none():
"""Defaults to None so existing /v1/.../render responses are unchanged."""
req = GenerateRequest(token_ids=[1, 2, 3], sampling_params=SamplingParams())
assert req.token_offsets is None
def test_generate_request_token_offsets_survive_json_round_trip():
"""GenerateRequest crosses the disagg serialization boundary; the
tuple[int, int] offsets must survive model_dump and re-validate."""
req = GenerateRequest(
token_ids=[10, 20],
sampling_params=SamplingParams(),
token_offsets=[(0, 1), (1, 3)],
)
dumped = req.model_dump()
assert dumped["token_offsets"] == [(0, 1), (1, 3)]
# Re-validate from the dumped dict (sampling_params doesn't round-trip
# cleanly via dump, so re-inject a fresh instance).
again = GenerateRequest.model_validate(
{**dumped, "sampling_params": SamplingParams()}
)
assert again.token_offsets == [(0, 1), (1, 3)]
tests/entrypoints/scale_out/__init__.py
View File
tests/entrypoints/scale_out/test_factories.py
-125
View File
@@ -1,125 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import logging
from argparse import Namespace
import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
from vllm.entrypoints.launchers.api_server.routers import register_api_routers
from vllm.entrypoints.scale_out.factories import register_scale_out_api_routers
from vllm.entrypoints.scale_out.token_in_token_out.api_router import attach_router
from vllm.entrypoints.serve.middleware.authenticate import (
GUARDED_PREFIX,
AuthenticationMiddleware,
)
from vllm.tasks import SupportedTask
GENERATE_PATH = "/inference/v1/generate"
ABORT_PATH = "/inference/v1/abort_requests"
UNAUTHENTICATED_ABORT_PATH = "/abort_requests"
SCALE_OUT_PATHS = {GENERATE_PATH, ABORT_PATH}
def registered_paths(
supported_tasks: tuple[SupportedTask, ...],
*,
tokens_only: bool = False,
enable_scale_out: bool = False,
) -> set[str]:
app = FastAPI()
args = Namespace(
tokens_only=tokens_only,
enable_scale_out=enable_scale_out,
enable_fault_tolerance=False,
)
app.state.args = args
register_api_routers(args, app, supported_tasks)
return {route.path for route in app.routes}
@pytest.mark.parametrize("path", sorted(SCALE_OUT_PATHS))
@pytest.mark.parametrize("authorized", [False, True])
def test_disabled_scale_out_routes_explain_opt_in_without_parsing(path, authorized):
app = FastAPI()
register_scale_out_api_routers(app, ("generate",))
app.add_middleware(AuthenticationMiddleware, tokens=["secret"])
headers = {"Authorization": "Bearer secret"} if authorized else {}
client = TestClient(app)
response = client.post(path, content="not JSON", headers=headers)
assert response.status_code == (403 if authorized else 401)
if authorized:
assert "--enable-scale-out" in response.json()["error"]["message"]
assert response.json()["error"]["type"] == "PermissionDeniedError"
assert path not in app.openapi()["paths"]
assert client.post("/unknown", headers=headers).status_code == 404
def test_disabled_scale_out_routes_are_logged_at_info(caplog):
with caplog.at_level("INFO", logger="vllm.entrypoints.scale_out.factories"):
registered_paths(("generate",))
assert [
record.levelno
for record in caplog.records
if "--enable-scale-out" in record.message
] == [logging.INFO]
@pytest.mark.parametrize("tokens_only", [False, True])
def test_enabled_generate_routes_use_native_request_validation(tokens_only):
app = FastAPI()
app.state.args = Namespace(
tokens_only=tokens_only, enable_scale_out=not tokens_only
)
register_scale_out_api_routers(app, ("generate",))
response = TestClient(app).post(
GENERATE_PATH,
content="not JSON",
headers={"Content-Type": "application/json"},
)
assert response.status_code == 422
assert response.json()["detail"][0]["type"] == "json_invalid"
def test_scale_out_routes_can_be_enabled_for_generate_server():
paths = registered_paths(("generate",), enable_scale_out=True)
assert paths >= SCALE_OUT_PATHS
def test_tokens_only_mode_enables_generate_routes_when_flag_is_unset():
paths = registered_paths(("generate",), tokens_only=True)
assert paths >= {GENERATE_PATH, ABORT_PATH, UNAUTHENTICATED_ABORT_PATH}
def test_abort_route_without_tokens_only_requires_api_key():
"""A decode pool that loads a tokenizer serves generate without
--tokens-only and still needs to abort, but only behind --api-key."""
paths = registered_paths(("generate",), enable_scale_out=True, tokens_only=False)
assert ABORT_PATH in paths
assert ABORT_PATH.startswith(GUARDED_PREFIX)
assert UNAUTHENTICATED_ABORT_PATH not in paths
def test_unauthenticated_abort_route_is_not_registered_twice():
"""A pre-existing /abort_requests route must not be shadowed by a second
registration from the scale-out router."""
app = FastAPI()
app.state.args = Namespace(tokens_only=True)
@app.post(UNAUTHENTICATED_ABORT_PATH)
async def existing_abort_requests():
return None
attach_router(app)
paths = [route.path for route in app.routes]
assert paths.count(UNAUTHENTICATED_ABORT_PATH) == 1
assert ABORT_PATH in paths
tests/entrypoints/scale_out/token_in_token_out/__init__.py
View File
tests/entrypoints/scale_out/token_in_token_out/test_mm_serde.py
-419
View File
@@ -1,419 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Roundtrip tests for multimodal serde used by the
token_in_token_out generate endpoint.
"""
import pytest
import torch
from pydantic import ValidationError
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.entrypoints.scale_out.token_in_token_out.mm_features import (
extract_mm_features,
merge_mm_kwargs_items,
mm_kwargs_from_features,
)
from vllm.entrypoints.scale_out.token_in_token_out.mm_serde import (
decode_mm_kwargs_item,
encode_mm_kwargs_item,
)
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
GenerateRequest,
MultiModalFeatures,
PlaceholderRangeInfo,
)
from vllm.inputs import mm_input
from vllm.multimodal.inputs import (
MultiModalBatchedField,
MultiModalFieldElem,
MultiModalFlatField,
MultiModalKwargsItem,
MultiModalSharedField,
PlaceholderRange,
)
def test_mm_kwargs_item_roundtrip():
"""Full roundtrip test with all three field types and multiple dtypes."""
e1 = MultiModalFieldElem(
data=torch.zeros(1000, dtype=torch.bfloat16),
field=MultiModalBatchedField(),
)
e2 = MultiModalFieldElem(
data=torch.ones(100, dtype=torch.int32),
field=MultiModalSharedField(batch_size=4),
)
e3 = MultiModalFieldElem(
data=torch.randn(20, dtype=torch.float32),
field=MultiModalFlatField(slices=[slice(0, 10), slice(10, 20)], dim=0),
)
item = MultiModalKwargsItem({"pixel_values": e1, "grid_thw": e2, "embeds": e3})
encoded = encode_mm_kwargs_item(item)
# Encoded result is a base64 string
assert isinstance(encoded, str)
decoded = decode_mm_kwargs_item(encoded)
assert set(decoded.keys()) == {"pixel_values", "grid_thw", "embeds"}
assert torch.equal(item["pixel_values"].data, decoded["pixel_values"].data)
assert torch.equal(item["grid_thw"].data, decoded["grid_thw"].data)
assert torch.equal(item["embeds"].data, decoded["embeds"].data)
assert isinstance(decoded["pixel_values"].field, MultiModalBatchedField)
assert isinstance(decoded["grid_thw"].field, MultiModalSharedField)
assert isinstance(decoded["embeds"].field, MultiModalFlatField)
def test_mm_kwargs_item_none_data():
"""Roundtrip with None data field."""
elem = MultiModalFieldElem(
data=None,
field=MultiModalSharedField(batch_size=2),
)
item = MultiModalKwargsItem({"empty": elem})
encoded = encode_mm_kwargs_item(item)
decoded = decode_mm_kwargs_item(encoded)
assert decoded["empty"].data is None
assert isinstance(decoded["empty"].field, MultiModalSharedField)
def test_mm_kwargs_item_nested_tensors():
"""Roundtrip with nested tensor data."""
nested = [torch.randn(3, 4), torch.randn(5, 4)]
elem = MultiModalFieldElem(
data=nested,
field=MultiModalBatchedField(),
)
item = MultiModalKwargsItem({"nested": elem})
encoded = encode_mm_kwargs_item(item)
decoded = decode_mm_kwargs_item(encoded)
decoded_data = decoded["nested"].data
assert len(decoded_data) == 2
assert torch.equal(nested[0], decoded_data[0])
assert torch.equal(nested[1], decoded_data[1])
def test_mm_features_with_kwargs_data():
"""Test that MultiModalFeatures can carry serialized tensor data."""
elem = MultiModalFieldElem(
data=torch.randn(5, 3, dtype=torch.float32),
field=MultiModalBatchedField(),
)
item = MultiModalKwargsItem({"pixel_values": elem})
encoded = encode_mm_kwargs_item(item)
features = MultiModalFeatures(
mm_hashes={"image": ["abc123"]},
mm_placeholders={"image": [PlaceholderRangeInfo(offset=0, length=10)]},
kwargs_data={"image": [encoded]},
)
# JSON roundtrip
json_str = features.model_dump_json()
features2 = MultiModalFeatures.model_validate_json(json_str)
assert features2.mm_hashes == {"image": ["abc123"]}
assert features2.kwargs_data is not None
assert len(features2.kwargs_data["image"]) == 1
decoded = decode_mm_kwargs_item(features2.kwargs_data["image"][0])
assert torch.equal(elem.data, decoded["pixel_values"].data)
def _image_engine_input(
*,
pixel_keep_on_cpu: bool = False,
grid_keep_on_cpu: bool = True,
) -> tuple[object, MultiModalFieldElem, MultiModalFieldElem]:
pixel_values = MultiModalFieldElem(
data=torch.randn(5, 3, dtype=torch.float32),
field=MultiModalBatchedField(keep_on_cpu=pixel_keep_on_cpu),
)
image_grid_thw = MultiModalFieldElem(
data=torch.tensor([[1, 24, 24]], dtype=torch.int32),
field=MultiModalBatchedField(keep_on_cpu=grid_keep_on_cpu),
)
item = MultiModalKwargsItem(
{"pixel_values": pixel_values, "image_grid_thw": image_grid_thw}
)
engine_input = mm_input(
prompt_token_ids=[1, 2, 3],
mm_kwargs={"image": [item]},
mm_hashes={"image": ["abc123"]},
mm_placeholders={"image": [PlaceholderRange(offset=0, length=1)]},
)
return engine_input, pixel_values, image_grid_thw
def test_render_extracts_metadata_fields_separately():
"""Render metadata contains CPU fields but excludes encoder inputs."""
engine_input, pixel_values, image_grid_thw = _image_engine_input()
features = extract_mm_features(engine_input)
assert features is not None
assert features.mm_metadata is not None
features = MultiModalFeatures.model_validate_json(features.model_dump_json())
assert features.mm_metadata is not None
metadata = decode_mm_kwargs_item(features.mm_metadata["image"][0])
assert set(metadata) == {"image_grid_thw"}
assert torch.equal(metadata["image_grid_thw"].data, image_grid_thw.data)
assert features.kwargs_data is not None
full_data = decode_mm_kwargs_item(features.kwargs_data["image"][0])
assert set(full_data) == {"pixel_values", "image_grid_thw"}
assert torch.equal(full_data["pixel_values"].data, pixel_values.data)
def test_extract_without_kwargs_keeps_layout():
"""Layout-only callers get hashes and placeholders, no serialized tensors."""
engine_input, _pixel_values, _image_grid_thw = _image_engine_input()
full = extract_mm_features(engine_input)
layout = extract_mm_features(engine_input, mm_payload="none")
assert full is not None and layout is not None
assert layout.kwargs_data is None
assert layout.mm_metadata is None
assert layout.mm_hashes == full.mm_hashes
assert layout.mm_placeholders == full.mm_placeholders
_MESSAGES = [{"role": "user", "content": "hi"}]
@pytest.mark.parametrize(
("request_cls", "body"),
[
(ChatCompletionRequest, {"model": "m", "messages": _MESSAGES}),
(CompletionRequest, {"model": "m", "prompt": "hi"}),
(ResponsesRequest, {"model": "m", "input": "hi"}),
],
)
def test_return_mm_kwargs_is_alias_for_mm_payload(request_cls, body):
"""Clients of the deprecated bool keep getting layout-only responses."""
assert request_cls.model_validate(body).mm_payload == "tensors"
for return_mm_kwargs, expected in [(False, "none"), (True, "tensors")]:
request = request_cls.model_validate(
{**body, "return_mm_kwargs": return_mm_kwargs}
)
assert request.mm_payload == expected
explicit = request_cls.model_validate(
{**body, "return_mm_kwargs": False, "mm_payload": "tensors"}
)
assert explicit.mm_payload == "tensors"
with pytest.raises(ValidationError):
request_cls.model_validate({**body, "mm_payload": "source"})
def test_extract_includes_declared_placeholder_metadata_fields():
"""EC placeholder metadata is kept even when keep_on_cpu is unset."""
engine_input, _pixel_values, image_grid_thw = _image_engine_input(
grid_keep_on_cpu=False
)
features = extract_mm_features(
engine_input,
metadata_fields_for=lambda modality: (
{"image_grid_thw"} if modality == "image" else set()
),
)
assert features is not None
assert features.mm_metadata is not None
metadata = decode_mm_kwargs_item(features.mm_metadata["image"][0])
assert set(metadata) == {"image_grid_thw"}
assert torch.equal(metadata["image_grid_thw"].data, image_grid_thw.data)
def test_legacy_kwargs_only_generate_keeps_full_payload():
"""Old clients that omit mm_metadata still reconstruct full kwargs."""
engine_input, pixel_values, image_grid_thw = _image_engine_input()
rendered = extract_mm_features(engine_input)
assert rendered is not None
assert rendered.kwargs_data is not None
features = MultiModalFeatures(
mm_hashes=rendered.mm_hashes,
mm_placeholders=rendered.mm_placeholders,
kwargs_data=rendered.kwargs_data,
)
merged = mm_kwargs_from_features(features)
item = merged["image"][0]
assert item is not None
assert set(item) == {"pixel_values", "image_grid_thw"}
assert torch.equal(item["pixel_values"].data, pixel_values.data)
assert torch.equal(item["image_grid_thw"].data, image_grid_thw.data)
def test_metadata_only_generate_requires_ec_transfer_params():
engine_input, _, _ = _image_engine_input()
rendered = extract_mm_features(engine_input)
assert rendered is not None
payload = {
"token_ids": [1, 2, 3],
"sampling_params": {"max_tokens": 1},
"features": {
"mm_hashes": rendered.mm_hashes,
"mm_placeholders": {
modality: [p.model_dump() for p in ranges]
for modality, ranges in rendered.mm_placeholders.items()
},
"mm_metadata": rendered.mm_metadata,
},
}
try:
GenerateRequest.model_validate(payload)
raise AssertionError("expected metadata-only request without EC to fail")
except ValidationError as exc:
assert "ec_transfer_params" in str(exc)
payload["ec_transfer_params"] = {"image-hash": {"peer_host": "10.0.0.1"}}
request = GenerateRequest.model_validate(payload)
assert request.features is not None
assert request.features.kwargs_data is None
merged = mm_kwargs_from_features(request.features)
item = merged["image"][0]
assert item is not None
assert set(item) == {"image_grid_thw"}
@pytest.mark.parametrize("modality", ["image", "video"])
@pytest.mark.parametrize(
("has_metadata", "has_ec"),
[(True, False), (True, True), (False, False)],
ids=["metadata-without-ec", "metadata-with-ec", "cache-hit-without-ec"],
)
def test_mixed_items_require_ec_only_for_metadata_only(
modality: str, has_metadata: bool, has_ec: bool
):
"""A full item must not bypass the EC requirement for another item."""
engine_input, _, image_grid_thw = _image_engine_input()
rendered = extract_mm_features(engine_input)
assert rendered is not None
features = rendered.model_dump()
metadata = (
encode_mm_kwargs_item(
MultiModalKwargsItem({f"{modality}_grid_thw": image_grid_thw})
)
if has_metadata
else None
)
second_item = {
"mm_hashes": "second-item",
"mm_placeholders": {"offset": 2, "length": 1},
"kwargs_data": None,
"mm_metadata": metadata,
}
for key, value in second_item.items():
features[key].setdefault(modality, []).append(value)
payload = {
"token_ids": [1, 2, 3],
"sampling_params": {"max_tokens": 1},
"features": features,
}
if has_ec:
payload["ec_transfer_params"] = {"second-item": {"peer_host": "10.0.0.1"}}
if has_metadata and not has_ec:
with pytest.raises(ValidationError, match="ec_transfer_params"):
GenerateRequest.model_validate(payload)
else:
GenerateRequest.model_validate(payload)
def test_kwargs_and_metadata_generate_does_not_require_ec():
engine_input, pixel_values, image_grid_thw = _image_engine_input()
rendered = extract_mm_features(engine_input)
assert rendered is not None
request = GenerateRequest.model_validate(
{
"token_ids": [1, 2, 3],
"sampling_params": {"max_tokens": 1},
"features": rendered.model_dump(),
}
)
assert request.features is not None
merged = mm_kwargs_from_features(request.features)
item = merged["image"][0]
assert item is not None
assert set(item) == {"pixel_values", "image_grid_thw"}
assert torch.equal(item["pixel_values"].data, pixel_values.data)
assert torch.equal(item["image_grid_thw"].data, image_grid_thw.data)
def test_cache_hit_null_kwargs_keeps_metadata_slot():
pixel_values = MultiModalFieldElem(
data=torch.randn(5, 3, dtype=torch.float32),
field=MultiModalBatchedField(),
)
image_grid_thw = MultiModalFieldElem(
data=torch.tensor([[1, 24, 24]], dtype=torch.int32),
field=MultiModalBatchedField(keep_on_cpu=True),
)
item = MultiModalKwargsItem(
{"pixel_values": pixel_values, "image_grid_thw": image_grid_thw}
)
engine_input = mm_input(
prompt_token_ids=[1, 2, 3, 4],
mm_kwargs={"image": [item, None]},
mm_hashes={"image": ["hit", "cached"]},
mm_placeholders={
"image": [
PlaceholderRange(offset=0, length=1),
PlaceholderRange(offset=2, length=1),
]
},
)
features = extract_mm_features(engine_input)
assert features is not None
assert features.kwargs_data is not None
assert features.mm_metadata is not None
assert features.kwargs_data["image"][1] is None
assert features.mm_metadata["image"][1] is None
assert len(features.mm_metadata["image"]) == 2
metadata = decode_mm_kwargs_item(features.mm_metadata["image"][0])
assert torch.equal(metadata["image_grid_thw"].data, image_grid_thw.data)
def test_features_alignment_rejects_length_mismatch():
try:
MultiModalFeatures(
mm_hashes={"image": ["a", "b"]},
mm_placeholders={"image": [PlaceholderRangeInfo(offset=0, length=1)]},
)
except ValidationError:
return
raise AssertionError("expected placeholder length mismatch to fail")
def test_metadata_can_replace_or_extend_full_mm_data():
"""Metadata-only requests retain the fields required by the model."""
metadata_elem = MultiModalFieldElem(
data=torch.tensor([[1, 24, 24]], dtype=torch.int32),
field=MultiModalBatchedField(keep_on_cpu=True),
)
metadata_item = MultiModalKwargsItem({"image_grid_thw": metadata_elem})
pixel_elem = MultiModalFieldElem(
data=torch.randn(5, 3),
field=MultiModalBatchedField(),
)
full_item = MultiModalKwargsItem(
{"pixel_values": pixel_elem, "image_grid_thw": metadata_elem}
)
assert merge_mm_kwargs_items(None, metadata_item) == metadata_item
merged = merge_mm_kwargs_items(full_item, metadata_item)
assert merged is not None
assert set(merged) == {"pixel_values", "image_grid_thw"}
tests/entrypoints/scale_out/token_in_token_out/test_protocol.py
-266
View File
@@ -1,266 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for the token_in_token_out request/response protocol.
These tests intentionally avoid spinning up a server — they exercise the
pydantic validators on ``GenerateRequest`` directly so they run fast and
fail loudly if the validator semantics ever drift.
"""
import json
import pytest
from pydantic import TypeAdapter, ValidationError
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
DerenderChatRequest,
DerenderChatStreamRequest,
DerenderCompletionRequest,
DerenderCompletionStreamRequest,
GenerateRequest,
GenerateResponse,
GenerateStreamResponse,
GenerateTextResponse,
GenerateTextStreamResponse,
GenerateTokensChoice,
GenerateTokensResponse,
GenerateTokensStreamResponse,
MultiModalFeatures,
PlaceholderRangeInfo,
)
from vllm.sampling_params import SamplingParams
def _base_payload() -> dict:
return {"token_ids": [1, 2, 3], "sampling_params": {}}
def test_omitted_max_tokens_is_not_provided():
"""Body without ``max_tokens`` must surface as 'not provided' so the
server can apply its own default instead of the dataclass 16."""
req = GenerateRequest.model_validate(_base_payload())
# SamplingParams' dataclass default leaks through the parsed instance —
# this is exactly the bug the server-side defaulting works around.
assert req.sampling_params.max_tokens == 16
assert not req.is_sampling_param_provided("max_tokens")
def test_explicit_max_tokens_is_provided():
"""Even when the client picks the same value as the dataclass default,
it must register as explicitly set so the server won't override it."""
payload = _base_payload()
payload["sampling_params"] = {"max_tokens": 16}
req = GenerateRequest.model_validate(payload)
assert req.sampling_params.max_tokens == 16
assert req.is_sampling_param_provided("max_tokens")
payload["sampling_params"] = {"max_tokens": 256}
req = GenerateRequest.model_validate(payload)
assert req.sampling_params.max_tokens == 256
assert req.is_sampling_param_provided("max_tokens")
def test_other_fields_tracked_independently():
payload = _base_payload()
payload["sampling_params"] = {"temperature": 0.0}
req = GenerateRequest.model_validate(payload)
assert not req.is_sampling_param_provided("max_tokens")
assert req.is_sampling_param_provided("temperature")
def test_json_roundtrip_preserves_provided_keys():
payload = _base_payload()
payload["sampling_params"] = {"temperature": 0.5}
req = GenerateRequest.model_validate_json(json.dumps(payload))
assert not req.is_sampling_param_provided("max_tokens")
assert req.is_sampling_param_provided("temperature")
def test_internal_instance_construction_treats_all_as_provided():
"""When internal callers build ``GenerateRequest`` from a pre-resolved
``SamplingParams`` instance, every field is considered explicitly set
so server-side defaulting can't clobber values resolved upstream."""
sp = SamplingParams(max_tokens=500, temperature=0.0)
req = GenerateRequest(token_ids=[1, 2, 3], sampling_params=sp)
assert req.is_sampling_param_provided("max_tokens")
assert req.is_sampling_param_provided("temperature")
# And keys we never touched should also count as provided in this path.
assert req.is_sampling_param_provided("top_p")
def test_multimodal_features_reject_mismatched_parallel_fields():
with pytest.raises(ValueError, match="same length"):
MultiModalFeatures(
mm_hashes={"image": ["a", "b"]},
mm_placeholders={"image": [PlaceholderRangeInfo(offset=0, length=1)]},
kwargs_data={"image": ["encoded"]},
)
def test_multimodal_features_reject_overlapping_placeholders():
with pytest.raises(ValueError, match="non-overlapping"):
MultiModalFeatures(
mm_hashes={"image": ["a", "b"]},
mm_placeholders={
"image": [
PlaceholderRangeInfo(offset=1, length=2),
PlaceholderRangeInfo(offset=2, length=2),
]
},
kwargs_data={"image": ["a", "b"]},
)
def test_generate_request_rejects_placeholder_outside_prompt():
with pytest.raises(ValueError, match="within the token_ids sequence"):
GenerateRequest(
token_ids=[1, 2, 3],
sampling_params=SamplingParams(),
features=MultiModalFeatures(
mm_hashes={"image": ["a"]},
mm_placeholders={"image": [PlaceholderRangeInfo(offset=2, length=2)]},
kwargs_data={"image": ["encoded"]},
),
)
def test_reasoning_parser_kwargs_rejects_unknown_keys():
"""The engine splats these into its reasoning parser's constructor, so a
client must not be able to pass arbitrary keyword arguments."""
payload = {**_base_payload(), "reasoning_parser_kwargs": {"tokenizer": None}}
with pytest.raises(ValidationError, match="tokenizer"):
GenerateRequest.model_validate(payload)
def test_rust_render_sampling_params_without_nulls_validate():
"""Rust render omits unset sampling fields; Python generate rejects nulls."""
sampling_params = {"max_tokens": 64, "watermarking": True}
request = GenerateRequest.model_validate(
{"token_ids": [1, 2, 3], "sampling_params": sampling_params}
)
assert request.sampling_params.max_tokens == 64
with pytest.raises(ValidationError, match="temperature"):
GenerateRequest.model_validate(
{
"token_ids": [1, 2, 3],
"sampling_params": {**sampling_params, "temperature": None},
}
)
def test_output_mode_defaults_to_tokens():
assert GenerateRequest.model_validate(_base_payload()).output_mode == "tokens"
def test_output_mode_text_is_accepted():
payload = {**_base_payload(), "output_mode": "text"}
assert GenerateRequest.model_validate(payload).output_mode == "text"
@pytest.mark.parametrize("output_mode", ["derender", "bogus", None])
def test_output_mode_rejects_unsupported_levels(output_mode):
"""Levels the server does not implement fail instead of falling back to
tokens which would answer 200 with the wrong response shape."""
payload = {**_base_payload(), "output_mode": output_mode}
with pytest.raises(ValidationError, match="output_mode"):
GenerateRequest.model_validate(payload)
def test_text_mode_rejects_detokenize_false():
payload = {
"token_ids": [1, 2, 3],
"sampling_params": {"detokenize": False},
"output_mode": "text",
}
with pytest.raises(ValidationError, match="detokenize"):
GenerateRequest.model_validate(payload)
def test_tokens_mode_allows_detokenize_false():
payload = {
"token_ids": [1, 2, 3],
"sampling_params": {"detokenize": False},
"output_mode": "tokens",
}
assert GenerateRequest.model_validate(payload).output_mode == "tokens"
def test_response_without_output_mode_parses_as_tokens():
"""Existing clients and older servers never send output_mode."""
parsed = TypeAdapter(GenerateResponse).validate_python(
{"choices": [{"index": 0, "token_ids": [1]}]}
)
assert isinstance(parsed, GenerateTokensResponse)
assert parsed.output_mode == "tokens"
def test_stream_chunk_without_output_mode_parses_as_tokens():
parsed = TypeAdapter(GenerateStreamResponse).validate_python(
{"choices": [{"index": 0, "token_ids": [1]}]}
)
assert isinstance(parsed, GenerateTokensStreamResponse)
def test_text_response_and_chunk_dispatch_on_output_mode():
response = TypeAdapter(GenerateResponse).validate_python(
{"output_mode": "text", "choices": [{"index": 0, "text": "hi"}]}
)
chunk = TypeAdapter(GenerateStreamResponse).validate_python(
{"output_mode": "text", "choices": [{"index": 0, "text": "hi"}]}
)
assert isinstance(response, GenerateTextResponse)
assert isinstance(chunk, GenerateTextStreamResponse)
def test_text_response_requires_text_on_every_choice():
with pytest.raises(ValidationError, match="text"):
TypeAdapter(GenerateResponse).validate_python(
{"output_mode": "text", "choices": [{"index": 0}]}
)
@pytest.mark.parametrize("output_mode", ["derender", None, 5])
def test_response_rejects_unknown_output_mode(output_mode):
with pytest.raises(ValidationError):
TypeAdapter(GenerateResponse).validate_python(
{"output_mode": output_mode, "choices": []}
)
def test_derender_requests_accept_responses_without_output_mode():
DerenderChatRequest.model_validate(
{"generate_response": {"choices": [{"index": 0, "token_ids": [1]}]}}
)
DerenderChatStreamRequest.model_validate(
{
"stream": True,
"generate_chunk": {"choices": [{"index": 0, "token_ids": [1]}]},
}
)
def test_derender_requests_reject_text_responses():
"""Text responses are already detokenized with stop strings applied, so
re-decoding their token_ids would silently undo the truncation."""
text = {"output_mode": "text", "choices": [{"index": 0, "text": "hi"}]}
with pytest.raises(ValidationError, match="output_mode"):
DerenderChatRequest.model_validate({"generate_response": text})
with pytest.raises(ValidationError, match="output_mode"):
DerenderCompletionRequest.model_validate({"generate_responses": [text]})
with pytest.raises(ValidationError, match="output_mode"):
DerenderChatStreamRequest.model_validate(
{"stream": True, "generate_chunk": text}
)
with pytest.raises(ValidationError, match="output_mode"):
DerenderCompletionStreamRequest.model_validate(
{"stream": True, "generate_chunk": text}
)
def test_tokens_response_echoes_output_mode_without_text():
dumped = GenerateTokensResponse(
choices=[GenerateTokensChoice(index=0, token_ids=[1])]
).model_dump()
assert dumped["output_mode"] == "tokens"
assert "text" not in dumped["choices"][0]
tests/entrypoints/scale_out/token_in_token_out/test_serving_multimodal_tokens.py
-262
View File
@@ -1,262 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for multimodal features through the /inference/v1/generate endpoint.
Mirrors test_serving_tokens.py but exercises the multimodal piping
using Qwen/Qwen3-VL-2B-Instruct end-to-end via the server's /render ->
/generate -> /detokenize path. Intentionally avoids running the HF
processor in the pytest parent process to keep os.fork() in other
tests (e.g. entrypoints/rl/weight_transfer/test_offline.py) deadlock-free.
"""
import httpx
import pytest
import pytest_asyncio
from PIL import Image
from tests.utils import RemoteOpenAIServer
from vllm.multimodal.utils import encode_image_url
MODEL_NAME = "Qwen/Qwen3-VL-2B-Instruct"
GEN_ENDPOINT = "/inference/v1/generate"
RENDER_ENDPOINT = "/v1/chat/completions/render"
DETOKENIZE_ENDPOINT = "/detokenize"
@pytest.fixture(scope="module")
def test_image():
return Image.new("RGB", (224, 224), color=(255, 0, 0))
@pytest.fixture(scope="module")
def server():
args = [
"--dtype",
"bfloat16",
"--max-model-len",
"4096",
"--enforce-eager",
"--no-enable-prefix-caching",
"--enable-scale-out",
]
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
yield remote_server
@pytest_asyncio.fixture
async def client(server: RemoteOpenAIServer):
transport = httpx.AsyncHTTPTransport(uds=server.uds) if server.uds else None
headers = {"Authorization": f"Bearer {server.DUMMY_API_KEY}"}
async with httpx.AsyncClient(
transport=transport,
base_url=server.url_root,
timeout=600,
headers=headers,
) as c:
yield c
@pytest.mark.asyncio
async def test_render_to_generate_roundtrip(client, test_image):
"""End-to-end: render a multimodal chat -> feed into generate -> decode.
All preprocessing and detokenization happens in the server subprocess;
the pytest parent never imports transformers or touches torch tensors.
"""
data_url = encode_image_url(test_image, format="PNG")
render_payload = {
"model": MODEL_NAME,
"messages": [
{
"role": "user",
"content": [
{"type": "image_url", "image_url": {"url": data_url}},
{
"type": "text",
"text": "What color is this image? Answer in one word.",
},
],
}
],
}
render_resp = await client.post(RENDER_ENDPOINT, json=render_payload)
render_resp.raise_for_status()
render_data = render_resp.json()
# Validate render output structure: keys exist and values are non-empty
# and well-typed.
assert "token_ids" in render_data
assert isinstance(render_data["token_ids"], list)
assert len(render_data["token_ids"]) > 0
assert all(isinstance(t, int) for t in render_data["token_ids"])
assert "features" in render_data
features = render_data["features"]
assert features is not None
assert isinstance(features, dict)
assert "mm_hashes" in features
assert "image" in features["mm_hashes"]
image_hashes = features["mm_hashes"]["image"]
assert isinstance(image_hashes, list)
assert len(image_hashes) > 0
assert all(isinstance(h, str) and h for h in image_hashes)
assert "mm_placeholders" in features
assert "image" in features["mm_placeholders"]
image_placeholders = features["mm_placeholders"]["image"]
assert isinstance(image_placeholders, list)
assert len(image_placeholders) > 0
for p in image_placeholders:
assert isinstance(p.get("offset"), int)
assert isinstance(p.get("length"), int)
assert p["length"] > 0
assert "kwargs_data" in features
assert "image" in features["kwargs_data"]
assert len(features["kwargs_data"]["image"]) > 0
# Build generate request from render output
generate_payload = render_data
generate_payload["sampling_params"] = {
"max_tokens": 10,
"temperature": 0.0,
}
gen_resp = await client.post(GEN_ENDPOINT, json=generate_payload)
gen_resp.raise_for_status()
gen_data = gen_resp.json()
assert "choices" in gen_data
assert isinstance(gen_data["choices"], list)
assert len(gen_data["choices"]) >= 1
choice = gen_data["choices"][0]
assert "token_ids" in choice
assert isinstance(choice["token_ids"], list)
assert len(choice["token_ids"]) > 0
assert all(isinstance(t, int) for t in choice["token_ids"])
detok_resp = await client.post(
DETOKENIZE_ENDPOINT,
json={"model": MODEL_NAME, "tokens": choice["token_ids"]},
)
detok_resp.raise_for_status()
detok_data = detok_resp.json()
assert "prompt" in detok_data
text = detok_data["prompt"]
assert isinstance(text, str)
assert len(text) > 0
assert "red" in text.lower(), (
f"Expected model to identify the red image, got: {text!r}"
)
def _collapse_placeholders(
token_ids: list[int], mm_placeholders: dict[str, list[dict[str, int]]]
) -> list[int]:
"""Undo placeholder expansion: each expanded range becomes one token.
``/generate`` with ``content_parts`` expects unexpanded placeholders and
expands them itself, so the roundtrip must start from a collapsed prompt.
"""
ranges = sorted(
(r["offset"], r["length"]) for rs in mm_placeholders.values() for r in rs
)
collapsed: list[int] = []
pos = 0
for offset, length in ranges:
collapsed.extend(token_ids[pos:offset])
collapsed.append(token_ids[offset])
pos = offset + length
collapsed.extend(token_ids[pos:])
return collapsed
async def _render_image_prompt(client, data_url: str) -> dict:
render_resp = await client.post(
RENDER_ENDPOINT,
json={
"model": MODEL_NAME,
"messages": [
{
"role": "user",
"content": [
{"type": "image_url", "image_url": {"url": data_url}},
{"type": "text", "text": "What color is this?"},
],
}
],
},
)
render_resp.raise_for_status()
return render_resp.json()
@pytest.mark.asyncio
async def test_content_parts_generates_tokens(client, test_image):
"""Unexpanded prompt + content_parts must expand exactly like /render."""
data_url = encode_image_url(test_image, format="PNG")
render_data = await _render_image_prompt(client, data_url)
expanded = render_data["token_ids"]
mm_placeholders = render_data["features"]["mm_placeholders"]
unexpanded = _collapse_placeholders(expanded, mm_placeholders)
assert len(unexpanded) < len(expanded)
gen_resp = await client.post(
GEN_ENDPOINT,
json={
"token_ids": unexpanded,
"content_parts": [{"type": "image_url", "url": data_url}],
"sampling_params": {"max_tokens": 10, "temperature": 0.0},
"return_token_ids": True,
},
)
gen_resp.raise_for_status()
gen_data = gen_resp.json()
assert "choices" in gen_data
choice = gen_data["choices"][0]
assert "token_ids" in choice
assert len(choice["token_ids"]) > 0
assert gen_data["prompt_token_ids"] == expanded
assert gen_data["mm_placeholders"] == mm_placeholders
@pytest.mark.asyncio
async def test_content_parts_streaming(client, test_image):
"""content_parts should work with streaming."""
import json as json_mod
data_url = encode_image_url(test_image, format="PNG")
render_data = await _render_image_prompt(client, data_url)
expanded = render_data["token_ids"]
mm_placeholders = render_data["features"]["mm_placeholders"]
unexpanded = _collapse_placeholders(expanded, mm_placeholders)
async with client.stream(
"POST",
GEN_ENDPOINT,
json={
"token_ids": unexpanded,
"content_parts": [{"type": "image_url", "url": data_url}],
"sampling_params": {"max_tokens": 8, "temperature": 0.0},
"stream": True,
"return_token_ids": True,
},
) as resp:
resp.raise_for_status()
chunks = []
async for line in resp.aiter_lines():
if line.startswith("data: ") and line != "data: [DONE]":
chunks.append(json_mod.loads(line[6:]))
assert len(chunks) > 0
prompt_metadata_chunks = [
chunk for chunk in chunks if chunk.get("prompt_token_ids") is not None
]
assert len(prompt_metadata_chunks) == 1
assert prompt_metadata_chunks[0]["prompt_token_ids"] == expanded
assert prompt_metadata_chunks[0]["mm_placeholders"] == mm_placeholders
tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py
-843
View File
@@ -1,843 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import io
import json
import httpx
import numpy as np
import pybase64 as base64
import pytest
import pytest_asyncio
from transformers import AutoTokenizer
from tests.utils import RemoteOpenAIServer
from vllm.config import ModelConfig
from vllm.v1.engine.detokenizer import check_stop_strings
MODEL_NAME = "Qwen/Qwen3-0.6B"
GEN_ENDPOINT = "/inference/v1/generate"
ABORT_ENDPOINT = "/inference/v1/abort_requests"
def get_vocab_size(model_name):
config = ModelConfig(
model=model_name,
seed=0,
dtype="bfloat16",
)
return config.get_vocab_size()
@pytest.fixture(scope="module")
def tokenizer():
return AutoTokenizer.from_pretrained(MODEL_NAME)
@pytest.fixture(scope="module")
def messages():
return [
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": "How many countries are in the EU?"},
]
@pytest.fixture(scope="module")
def server(request):
args = [
"--dtype",
"bfloat16",
"--max-model-len",
"1024",
"--enforce-eager",
# On ROCm (e.g. MI355X/gfx950), bf16 GEMM results can differ by
# 1 ULP when the batch dimension (M) changes, because different M
# values cause the Tensile backend to select different tile
# configurations with different fp32 accumulation orders. With
# prefix caching, cache-miss prefills compute all tokens in one
# pass (large M) while cache-hit requests compute only the
# uncached suffix (small M), seeding a divergence that amplifies
# through the residual stream and flips argmax tokens.
# See: https://github.com/vllm-project/vllm/issues/33123
#
# Either disable prefix caching entirely, or enable it with
# --deterministic-prefix-caching which forces cache-miss prefills
# to split at block boundaries so the suffix GEMM shape is always
# identical regardless of cache state.
#
# Option A: disable prefix caching
"--no-enable-prefix-caching",
#
# Option B: deterministic prefix caching
# "--enable-prefix-caching",
# "--deterministic-prefix-caching",
"--enable-scale-out",
]
extra_args = getattr(request, "param", None)
if extra_args is not None:
args = args + (
list(extra_args)
if isinstance(extra_args, (list, tuple))
else [str(extra_args)]
)
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
yield remote_server
@pytest_asyncio.fixture
async def client(server: RemoteOpenAIServer):
transport = httpx.AsyncHTTPTransport(uds=server.uds) if server.uds else None
headers = {"Authorization": f"Bearer {server.DUMMY_API_KEY}"}
async with httpx.AsyncClient(
transport=transport,
base_url=server.url_root,
timeout=600,
headers=headers,
) as c:
yield c
@pytest.mark.asyncio
async def test_generate_endpoint(client):
payload = {
"model": MODEL_NAME,
"token_ids": [1, 2, 3],
"sampling_params": {"max_tokens": 5},
"stream": False,
}
resp = await client.post(GEN_ENDPOINT, json=payload)
resp.raise_for_status()
data = resp.json()
assert "choices" in data
assert data["choices"][0].get("sampling_mask") is None
@pytest.mark.asyncio
async def test_generate_rejects_min_tokens_above_filled_max_tokens(client):
"""Explicit null max_tokens must not skip the min_tokens bound."""
payload = {
"model": MODEL_NAME,
"token_ids": [1, 2, 3],
"sampling_params": {"max_tokens": None, "min_tokens": 2147483648},
"stream": False,
}
resp = await client.post(GEN_ENDPOINT, json=payload)
assert resp.status_code == 400
assert "min_tokens" in resp.json()["error"]["message"]
followup = {
"model": MODEL_NAME,
"token_ids": [1, 2, 3],
"sampling_params": {"max_tokens": 4},
"stream": False,
}
resp = await client.post(GEN_ENDPOINT, json=followup)
resp.raise_for_status()
@pytest.mark.asyncio
@pytest.mark.skipif(
False,
reason="sampling mask output is not supported by the Rust frontend",
)
@pytest.mark.parametrize(
"server",
[["--return-sampling-mask", "--logprobs-mode", "processed_logprobs"]],
indirect=True,
)
async def test_generate_sampling_mask(client):
top_k = 5
payload = {
"model": MODEL_NAME,
"token_ids": [1, 2, 3],
"sampling_params": {
"max_tokens": 5,
"temperature": 0.8,
"top_k": top_k,
"top_p": 0.9,
"ignore_eos": True,
"seed": 0,
"logprobs": top_k,
},
"stream": False,
}
resp = await client.post(GEN_ENDPOINT, json=payload)
resp.raise_for_status()
choice = resp.json()["choices"][0]
token_ids = choice["token_ids"]
sampling_mask = choice["sampling_mask"]
assert sampling_mask is not None
assert len(token_ids) == len(sampling_mask)
vocab_size = get_vocab_size(MODEL_NAME)
logprobs_content = choice["logprobs"]["content"]
for token_id, support, entry in zip(token_ids, sampling_mask, logprobs_content):
assert support
assert len(support) == len(set(support))
assert all(0 <= support_token_id < vocab_size for support_token_id in support)
assert token_id in support
# processed_logprobs: exactly the support carries finite probability.
for top in entry["top_logprobs"]:
in_support = top["token_id"] in support
assert in_support == (top["logprob"] > -9999.0)
@pytest.mark.asyncio
async def test_generate_defaults_max_tokens_when_omitted(client):
"""Regression: omitting ``max_tokens`` must not silently cap at 16.
``SamplingParams.max_tokens`` defaults to 16. Before the server-side
defaulting was wired up, every request that didn't set ``max_tokens``
truncated mid-generation. The server should now fill it in from
``max_model_len - prompt_len`` (matching ``/v1/chat/completions``).
"""
payload = {
"model": MODEL_NAME,
"token_ids": [1, 2, 3],
"sampling_params": {
"temperature": 0.0,
"ignore_eos": True,
},
"stream": False,
}
resp = await client.post(GEN_ENDPOINT, json=payload)
resp.raise_for_status()
data = resp.json()
completion_tokens = len(data["choices"][0]["token_ids"])
# max_model_len=1024 in the test fixture, prompt is 3 tokens, so we
# should get ~1021 tokens of output (capped at max_model_len boundary).
assert completion_tokens > 16, (
f"expected server-side default to exceed the legacy 16-token cap, "
f"got {completion_tokens}"
)
@pytest.mark.asyncio
@pytest.mark.skipif(
False,
reason="parallel sampling (n > 1) is not supported by the Rust frontend",
)
async def test_generate_returns_all_choices_when_n_greater_than_one(client):
"""Regression: ``n > 1`` must return ``n`` choices.
Non-streaming requests kept ``SamplingParams``' default output kind,
``CUMULATIVE``, so only the sequences updated during the last engine step
reached the response and the others were silently dropped.
"""
payload = {
"model": MODEL_NAME,
"token_ids": [1, 2, 3],
"sampling_params": {"max_tokens": 5, "temperature": 1.0, "n": 4},
"stream": False,
}
resp = await client.post(GEN_ENDPOINT, json=payload)
resp.raise_for_status()
data = resp.json()
assert sorted(choice["index"] for choice in data["choices"]) == [0, 1, 2, 3]
@pytest.mark.asyncio
async def test_generate_stream(client):
payload = {
"model": MODEL_NAME,
"token_ids": [1, 2, 3],
"sampling_params": {"max_tokens": 5},
"stream": True,
}
async with client.stream("POST", GEN_ENDPOINT, json=payload) as resp:
resp.raise_for_status()
chunks = []
async for line in resp.aiter_lines():
if not line.startswith("data: "):
continue
payload_str = line[len("data: ") :]
if payload_str == "[DONE]":
break
chunks.append(json.loads(payload_str))
assert len(chunks) > 0
# Every chunk has choices with token_ids
all_token_ids = []
for chunk in chunks:
assert "choices" in chunk
assert len(chunk["choices"]) == 1
choice = chunk["choices"][0]
assert "token_ids" in choice
assert len(choice["token_ids"]) > 0
all_token_ids.extend(choice["token_ids"])
# Last chunk should have a finish_reason
assert chunks[-1]["choices"][0]["finish_reason"] is not None
# Streaming should produce the same tokens as non-streaming
non_stream_resp = await client.post(
GEN_ENDPOINT,
json={
"model": MODEL_NAME,
"token_ids": [1, 2, 3],
"sampling_params": {"max_tokens": 5, "temperature": 0.0},
"stream": False,
},
)
non_stream_data = non_stream_resp.json()
# Just verify we got the right number of tokens
assert len(all_token_ids) == len(non_stream_data["choices"][0]["token_ids"])
@pytest.mark.asyncio
@pytest.mark.parametrize("logprobs_value", [0, 1, 5])
async def test_generate_logprobs(client, logprobs_value):
payload = {
"model": MODEL_NAME,
"token_ids": [1, 2, 3],
"sampling_params": {
"max_tokens": 5,
"temperature": 0.0,
"logprobs": logprobs_value,
},
"stream": False,
}
resp = await client.post(GEN_ENDPOINT, json=payload)
resp.raise_for_status()
data = resp.json()
choice = data["choices"][0]
assert choice["logprobs"] is not None
logprobs_content = choice["logprobs"]["content"]
assert len(logprobs_content) == len(choice["token_ids"])
for entry in logprobs_content:
assert "logprob" in entry
assert len(entry["top_logprobs"]) >= 1
assert len(entry["top_logprobs"]) == max(logprobs_value, 1)
@pytest.mark.asyncio
async def test_generate_prompt_token_id_logprobs(client):
token_ids = [11, 22, 33, 44, 55]
rows = [[33], [44, 100], [55]]
sampling_params = {
"max_tokens": 1,
"prompt_logprobs": 0,
"prompt_logprob_token_ids": rows,
"prompt_logprob_start": 1,
}
payload = {
"model": MODEL_NAME,
"token_ids": token_ids,
"sampling_params": sampling_params,
}
resp = await client.post(GEN_ENDPOINT, json=payload)
resp.raise_for_status()
data = resp.json()
scores = np.load(io.BytesIO(base64.b64decode(data["prompt_token_id_logprobs"])))
assert scores.shape == (3, 2)
for row, target in enumerate(token_ids[2:]):
expected = data["prompt_logprobs"][row + 2][str(target)]["logprob"]
assert scores[row, 0] == pytest.approx(expected, abs=1e-3)
assert np.isneginf(scores[[0, 2], 1]).all()
stream_only_ids = {"prompt_logprob_token_ids": rows}
resp = await client.post(
GEN_ENDPOINT,
json={**payload, "sampling_params": stream_only_ids, "stream": True},
)
assert resp.status_code == 400
assert "prompt_logprob_token_ids" in resp.text
too_few_rows = {**sampling_params, "prompt_logprob_token_ids": rows[:2]}
resp = await client.post(
GEN_ENDPOINT, json={**payload, "sampling_params": too_few_rows}
)
assert resp.status_code == 400
assert "scored rows" in resp.text
@pytest.mark.asyncio
async def test_same_response_as_chat_completions(client, tokenizer, messages):
token_ids = tokenizer.apply_chat_template(
messages,
add_generation_prompt=True,
enable_thinking=False, # default with Qwen3
return_dict=True, # default with Transformers v5
).input_ids
for ignore_eos in [True, False]:
payload = {
"model": MODEL_NAME,
"token_ids": token_ids,
"sampling_params": {
"max_tokens": 24,
"temperature": 0.0,
# NOTE coordinator will set this to skip detokenization
"detokenize": False,
"ignore_eos": ignore_eos,
},
"stream": False,
}
generate_resp = await client.post(GEN_ENDPOINT, json=payload)
generate_data = generate_resp.json()
gen_token_ids = generate_data["choices"][0]["token_ids"]
generate_res = tokenizer.decode(gen_token_ids, skip_special_tokens=True)
payload = {
"model": MODEL_NAME,
"messages": messages,
"max_tokens": 24,
"temperature": 0.0,
"stream": False,
"ignore_eos": ignore_eos,
"chat_template_kwargs": {"enable_thinking": False},
}
completions_resp = await client.post("/v1/chat/completions", json=payload)
completions_data = completions_resp.json()
completions_res = completions_data["choices"][0]["message"]["content"]
if ignore_eos:
# When ignoring EOS, only compare up to the first EOS token
# Post-EOS generation is undefined and may differ
eos_tokens = {
tokenizer.eos_token_id,
*tokenizer.extra_special_tokens_ids,
}
# Find first EOS in generated tokens
eos_pos = None
for i, tid in enumerate(gen_token_ids):
if tid in eos_tokens:
eos_pos = i
break
if eos_pos is not None:
gen_token_ids_truncated = gen_token_ids[:eos_pos]
generate_res = tokenizer.decode(
gen_token_ids_truncated, skip_special_tokens=True
)
# Truncate completions_res to same length for comparison
completions_res = completions_res[: len(generate_res)]
assert generate_res == completions_res
@pytest.mark.asyncio
async def test_stop_string_workflow(client, tokenizer, messages):
token_ids = tokenizer.apply_chat_template(
messages,
add_generation_prompt=True,
enable_thinking=False, # default with Qwen3
return_dict=True, # default with Transformers v5
).input_ids
payload = {
"model": MODEL_NAME,
"token_ids": token_ids,
"sampling_params": {
"max_tokens": 24,
"temperature": 0.0,
"detokenize": False,
# stop strings are only supported when detokenize is True.
"stop": ["27 member"],
},
# TODO stream test is much more interesting
"stream": False,
}
with pytest.raises(httpx.HTTPStatusError):
generate_resp = await client.post(GEN_ENDPOINT, json=payload)
generate_resp.raise_for_status()
payload["sampling_params"]["stop"] = None
generate_resp = await client.post(
GEN_ENDPOINT, json=payload, headers={"X-Request-Id": "42"}
)
generate_data = generate_resp.json()
generate_res = tokenizer.decode(
generate_data["choices"][0]["token_ids"], skip_special_tokens=True
)
# NOTE This is under the responsibility of the coordinator
# stop_checker = StopChecker(
# max_model_len=1024, get_tokenizer_for_seq=lambda _: tokenizer
# )
stop_str, truncate_to = check_stop_strings(
generate_res, len(generate_res), ["27 member"], False
)
assert stop_str == "27 member"
# abort request that hit stop string (requires tokens-only mode)
# res = await client.post("/abort_requests", json={"request_ids": ["generate-tokens-42"]}) # noqa: E501
# res.raise_for_status()
generate_res = generate_res[:truncate_to]
# Get stop_str response from chat completions
payload = {
"model": MODEL_NAME,
"messages": messages,
"max_tokens": 24,
"temperature": 0.0,
"stream": False,
"stop": ["27 member"],
"chat_template_kwargs": dict(enable_thinking=False),
}
completions_resp = await client.post("/v1/chat/completions", json=payload)
completions_data = completions_resp.json()
completions_res = completions_data["choices"][0]["message"]["content"]
assert generate_res == completions_res
@pytest.mark.asyncio
@pytest.mark.skipif(
False,
reason="--tokens-only is not supported by the Rust frontend",
)
@pytest.mark.parametrize("server", [["--tokens-only"]], indirect=True)
async def test_stop_strings_rejected_when_tokens_only(client):
"""--tokens-only forces detokenize=False after request validation, so
stop strings must be rejected explicitly rather than silently ignored."""
sampling_params = {"max_tokens": 5, "stop": ["never"]}
payload = {
"model": MODEL_NAME,
"token_ids": [1, 2, 3],
"sampling_params": sampling_params,
"stream": False,
}
resp = await client.post(GEN_ENDPOINT, json=payload)
assert resp.status_code == 400
assert "stop strings" in resp.json()["error"]["message"]
sampling_params["stop"] = None
resp = await client.post(GEN_ENDPOINT, json=payload)
resp.raise_for_status()
@pytest.mark.asyncio
@pytest.mark.parametrize(
"server",
[
[
"--enable-lora",
"--lora-modules",
"Alice=charent/self_cognition_Alice",
"Bob=charent/self_cognition_Bob",
"--max-lora-rank",
"64",
"--max-cpu-loras",
"2",
]
],
indirect=True,
)
async def test_generate_with_lora_adapter(client, tokenizer, messages):
# Verify adapters are listed
models_resp = await client.get("/v1/models")
models_resp.raise_for_status()
models = {m["id"] for m in models_resp.json().get("data", [])}
assert {"Alice", "Bob"}.issubset(models)
# Generate using a LoRA adapter by specifying its name as the model
payload = {
"model": "Alice",
"token_ids": [1, 2, 3],
"sampling_params": {"max_tokens": 5},
"stream": False,
}
resp = await client.post(GEN_ENDPOINT, json=payload)
resp.raise_for_status()
data = resp.json()
assert "choices" in data
token_ids = tokenizer.apply_chat_template(
messages,
add_generation_prompt=True,
enable_thinking=False, # default with Qwen3
return_dict=True, # default with Transformers v5
).input_ids
payload = {
"model": "Alice",
"token_ids": token_ids,
"sampling_params": {
"max_tokens": 24,
"temperature": 0.0,
"detokenize": False,
},
"stream": False,
}
generate_resp = await client.post(GEN_ENDPOINT, json=payload)
generate_data = generate_resp.json()
generate_res = tokenizer.decode(
generate_data["choices"][0]["token_ids"], skip_special_tokens=True
)
payload = {
"model": "Alice",
"messages": messages,
"max_tokens": 24,
"temperature": 0.0,
"stream": False,
"chat_template_kwargs": dict(enable_thinking=False),
}
completions_resp = await client.post("/v1/chat/completions", json=payload)
completions_data = completions_resp.json()
completions_res = completions_data["choices"][0]["message"]["content"]
assert generate_res == completions_res
def _structured_chat_body(messages, **overrides) -> dict:
return {
"model": MODEL_NAME,
"messages": messages,
"max_tokens": 64,
"temperature": 0.0,
"chat_template_kwargs": {"enable_thinking": True},
"response_format": {
"type": "json_schema",
"json_schema": {
"name": "answer",
"schema": {
"type": "object",
"properties": {"count": {"type": "integer"}},
"required": ["count"],
},
},
},
**overrides,
}
async def _render_then_generate(client, body) -> tuple[dict, list[int]]:
render_resp = await client.post("/v1/chat/completions/render", json=body)
render_resp.raise_for_status()
generate_request = render_resp.json()
generate_resp = await client.post(GEN_ENDPOINT, json=generate_request)
generate_resp.raise_for_status()
return generate_request, generate_resp.json()["choices"][0]["token_ids"]
# deepseek_v3 reads `enable_thinking` from the template kwargs and uses Qwen3's
# <think> tokens, so the engine gates structured outputs differently when
# either reasoning field is dropped between render and generate.
@pytest.mark.asyncio
@pytest.mark.skipif(
False,
reason="the Rust frontend does not forward reasoning fields to the engine",
)
@pytest.mark.parametrize(
"server", [["--reasoning-parser", "deepseek_v3"]], indirect=True
)
async def test_render_then_generate_forwards_reasoning_parser_kwargs(client, messages):
"""With thinking on, structured outputs must wait for the reasoning to
end on generate, the same as on chat."""
body = _structured_chat_body(messages, return_token_ids=True)
chat_resp = await client.post("/v1/chat/completions", json=body)
chat_resp.raise_for_status()
generate_request, token_ids = await _render_then_generate(client, body)
assert generate_request["reasoning_ended"] is False
assert token_ids == chat_resp.json()["choices"][0]["token_ids"]
@pytest.mark.asyncio
@pytest.mark.skipif(
False,
reason="the Rust frontend does not forward reasoning fields to the engine",
)
@pytest.mark.parametrize(
"server", [["--reasoning-parser", "deepseek_v3"]], indirect=True
)
async def test_render_then_generate_forwards_reasoning_ended(
client, tokenizer, messages
):
"""`include_reasoning=false` constrains from the first token on chat, so
generate must too, even though the engine's parser expects thinking."""
body = _structured_chat_body(messages, include_reasoning=False)
generate_request, token_ids = await _render_then_generate(client, body)
assert generate_request["reasoning_ended"] is True
output = json.loads(tokenizer.decode(token_ids, skip_special_tokens=True))
assert isinstance(output["count"], int)
def _chat_prompt_token_ids(tokenizer, messages) -> list[int]:
return tokenizer.apply_chat_template(
messages,
add_generation_prompt=True,
enable_thinking=False, # default with Qwen3
return_dict=True, # default with Transformers v5
).input_ids
def _text_mode_payload(token_ids, **sampling_params) -> dict:
return {
"model": MODEL_NAME,
"token_ids": token_ids,
"sampling_params": {"max_tokens": 24, "temperature": 0.0, **sampling_params},
"output_mode": "text",
"stream": False,
}
@pytest.mark.asyncio
@pytest.mark.skipif(
False,
reason="output_mode is not supported by the Rust frontend",
)
async def test_text_mode_matches_chat_completions(client, tokenizer, messages):
token_ids = _chat_prompt_token_ids(tokenizer, messages)
resp = await client.post(GEN_ENDPOINT, json=_text_mode_payload(token_ids))
resp.raise_for_status()
data = resp.json()
assert data["output_mode"] == "text"
payload = {
"model": MODEL_NAME,
"messages": messages,
"max_tokens": 24,
"temperature": 0.0,
"stream": False,
"chat_template_kwargs": {"enable_thinking": False},
}
chat_resp = await client.post("/v1/chat/completions", json=payload)
chat_content = chat_resp.json()["choices"][0]["message"]["content"]
assert data["choices"][0]["text"] == chat_content
assert data["choices"][0]["token_ids"]
@pytest.mark.asyncio
@pytest.mark.skipif(
False,
reason="output_mode is not supported by the Rust frontend",
)
async def test_text_mode_stream_matches_non_stream(client, tokenizer, messages):
token_ids = _chat_prompt_token_ids(tokenizer, messages)
payload = _text_mode_payload(token_ids)
non_stream = await client.post(GEN_ENDPOINT, json=payload)
non_stream.raise_for_status()
chunks = []
async with client.stream(
"POST", GEN_ENDPOINT, json={**payload, "stream": True}
) as resp:
resp.raise_for_status()
async for line in resp.aiter_lines():
if line.startswith("data: ") and line != "data: [DONE]":
chunks.append(json.loads(line[len("data: ") :]))
assert {chunk["output_mode"] for chunk in chunks} == {"text"}
assert chunks[-1]["choices"][0]["finish_reason"] is not None
streamed_text = "".join(chunk["choices"][0]["text"] for chunk in chunks)
assert streamed_text == non_stream.json()["choices"][0]["text"]
@pytest.mark.asyncio
@pytest.mark.skipif(
False,
reason="output_mode is not supported by the Rust frontend",
)
@pytest.mark.parametrize(
"overrides, expected",
[
pytest.param(
{"sampling_params": {"detokenize": False}}, "detokenize", id="no-detok"
),
pytest.param({"output_mode": "derender"}, "output_mode", id="unsupported"),
],
)
async def test_text_mode_invalid_requests_return_400(client, overrides, expected):
payload = {
**_text_mode_payload([1, 2, 3], max_tokens=5),
**overrides,
}
resp = await client.post(GEN_ENDPOINT, json=payload)
assert resp.status_code == 400
assert expected in resp.json()["error"]["message"]
@pytest.mark.asyncio
@pytest.mark.skipif(
False,
reason="--tokens-only is not supported by the Rust frontend",
)
@pytest.mark.parametrize("server", [["--tokens-only"]], indirect=True)
async def test_text_mode_rejected_when_tokens_only(client):
"""Without a tokenizer the detokenizer returns "", so the request must
fail instead of answering 200 with empty text."""
resp = await client.post(GEN_ENDPOINT, json=_text_mode_payload([1, 2, 3]))
assert resp.status_code == 400
assert "requires a tokenizer" in resp.json()["error"]["message"]
payload = {**_text_mode_payload([1, 2, 3]), "output_mode": "tokens"}
payload["sampling_params"]["detokenize"] = False
resp = await client.post(GEN_ENDPOINT, json=payload)
resp.raise_for_status()
@pytest.mark.asyncio
@pytest.mark.skipif(
False,
reason="the Rust frontend does not serve /inference/v1/abort_requests",
)
@pytest.mark.parametrize("output_mode", ["tokens", "text"])
async def test_stream_delivers_abort_finish_chunk(
client, tokenizer, messages, output_mode
):
"""The final output after an abort has no new token IDs, but the
stream must still deliver its finish reason."""
payload = {
**_text_mode_payload(
_chat_prompt_token_ids(tokenizer, messages),
max_tokens=900,
ignore_eos=True,
),
"output_mode": output_mode,
"stream": True,
"request_id": f"{output_mode}-abort-e2e",
}
chunks = []
aborted = False
async with client.stream("POST", GEN_ENDPOINT, json=payload) as resp:
resp.raise_for_status()
async for line in resp.aiter_lines():
if not line.startswith("data: ") or line == "data: [DONE]":
continue
chunk = json.loads(line[len("data: ") :])
chunks.append(chunk)
if not aborted and chunk["choices"] and chunk["choices"][0]["token_ids"]:
abort = await client.post(
ABORT_ENDPOINT,
json={"request_ids": [chunk["request_id"]]},
)
assert abort.status_code == 200
aborted = True
assert aborted
assert {chunk["output_mode"] for chunk in chunks} == {output_mode}
final_choice = chunks[-1]["choices"][0]
assert final_choice["finish_reason"] == "abort"
generated = sum(len(chunk["choices"][0]["token_ids"] or []) for chunk in chunks)
assert generated < 900
@pytest.mark.asyncio
@pytest.mark.skipif(
False,
reason="the Rust frontend does not serve /inference/v1/abort_requests",
)
async def test_abort_requests_is_served_without_tokens_only(client):
resp = await client.post(
ABORT_ENDPOINT, json={"request_ids": ["generate-tokens-unknown"]}
)
assert resp.status_code == 200
resp = await client.post(ABORT_ENDPOINT, json={})
assert resp.status_code == 400
# The unauthenticated path is only served with --tokens-only.
resp = await client.post(
"/abort_requests", json={"request_ids": ["generate-tokens-unknown"]}
)
assert resp.status_code == 404
tests/entrypoints/scale_out/token_in_token_out/test_tokens_logprobs.py
-175
View File
@@ -1,175 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from unittest.mock import MagicMock
import pytest
from pydantic import ValidationError
from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateLogProb
from vllm.entrypoints.scale_out.token_in_token_out.serving import ServingTokens
from vllm.logprobs import Logprob
def test_top_logprobs_alternatives_have_own_token_ids():
"""Each top_logprobs alternative must carry its own integer token id."""
# Engine dict at logprobs=2 with the sampled token 262 also top-1.
result = ServingTokens._create_tokens_logprobs(
None,
token_ids=[262],
top_logprobs=[{262: Logprob(-0.1), 257: Logprob(-1.2)}],
)
token_ids = {e.token_id for e in result.content[0].top_logprobs}
assert token_ids == {262, 257}, f"got {token_ids}"
def test_sampled_entry_carries_token_id_and_rank():
"""The sampled token is identified by id, and the engine's rank survives."""
result = ServingTokens._create_tokens_logprobs(
None,
token_ids=[262],
top_logprobs=[
{262: Logprob(-0.1, rank=1), 257: Logprob(-1.2, rank=2)},
],
)
entry = result.content[0]
assert entry.token_id == 262
assert entry.logprob == -0.1
assert entry.rank == 1
assert [(t.token_id, t.rank) for t in entry.top_logprobs] == [(262, 1), (257, 2)]
# No tokenizer on the generate server: no string token, no bytes.
assert not hasattr(entry, "token")
assert not hasattr(entry, "bytes")
def test_sampled_token_absent_from_topk_uses_sentinel():
"""A sampled token missing from the engine's map keeps the -9999.0 sentinel."""
result = ServingTokens._create_tokens_logprobs(
None,
token_ids=[5],
top_logprobs=[{7: Logprob(-0.9)}],
)
entry = result.content[0]
assert entry.token_id == 5
assert entry.logprob == -9999.0
assert entry.rank is None
assert entry.top_logprobs == []
def test_logprobs_zero_emits_sampled_token():
"""logprobs=0: the engine returns only the sampled token, and it is kept."""
result = ServingTokens._create_tokens_logprobs(
None,
token_ids=[7],
top_logprobs=[{7: Logprob(-0.9)}],
)
assert [t.token_id for t in result.content[0].top_logprobs] == [7]
def test_logprobs_minus_one_emits_all_tokens():
result = ServingTokens._create_tokens_logprobs(
None,
token_ids=[7],
top_logprobs=[{7: Logprob(-0.9), 8: Logprob(-1.1)}],
)
assert len(result.content[0].top_logprobs) == 2
def test_text_logprobs_without_tokenizer_are_placeholders_without_bytes():
"""--return-tokens-as-token-ids keeps `token_id:N` placeholders in text mode."""
result = ServingTokens._create_text_logprobs(
None,
token_ids=[7],
top_logprobs=[{7: Logprob(-0.9), 8: Logprob(-1.1)}],
)
entry = result.content[0]
assert entry.token == "token_id:7"
assert entry.bytes is None
assert all(t.bytes is None for t in entry.top_logprobs)
def test_resolved_logprobs_use_engine_decoded_tokens_and_bytes():
tokenizer = MagicMock()
result = ServingTokens._create_text_logprobs(
None,
token_ids=[7],
top_logprobs=[
{7: Logprob(-0.9, decoded_token="é"), 8: Logprob(-1.1, decoded_token="e")}
],
tokenizer=tokenizer,
)
entry = result.content[0]
assert (entry.token, entry.bytes) == ("é", [195, 169])
assert [(t.token, t.bytes) for t in entry.top_logprobs] == [
("é", [195, 169]),
("e", [101]),
]
tokenizer.decode.assert_not_called()
def test_resolved_logprobs_decode_tokens_the_engine_did_not():
tokenizer = MagicMock()
tokenizer.decode.side_effect = lambda ids: f"<{ids[0]}>"
result = ServingTokens._create_text_logprobs(
None,
token_ids=[7, 9],
top_logprobs=[{7: Logprob(-0.9)}, None],
tokenizer=tokenizer,
)
assert [e.token for e in result.content] == ["<7>", "<9>"]
assert result.content[0].top_logprobs[0].token == "<7>"
assert result.content[1].bytes == list(b"<9>")
def test_sampled_token_outside_topk_comes_first():
"""The engine puts the sampled token first, then ranks 1..k. Generate keeps
all k + 1 candidates when the sampled token is outside the top k; derender
applies each endpoint's cut (#59513)."""
result = ServingTokens._create_tokens_logprobs(
None,
token_ids=[50],
top_logprobs=[
{
50: Logprob(-3.0, rank=5),
10: Logprob(-0.2, rank=1),
20: Logprob(-1.0, rank=2),
}
],
)
assert [t.rank for t in result.content[0].top_logprobs] == [5, 1, 2]
def test_logprob_is_required_on_the_wire():
"""A payload missing logprob is rejected rather than read as -9999."""
with pytest.raises(ValidationError):
GenerateLogProb.model_validate({"token_id": 1})
def test_rank_zero_from_a_nan_logprob_is_none():
"""The engine reports rank 0 for a sampled token whose logprob is NaN."""
result = ServingTokens._create_tokens_logprobs(
None,
token_ids=[7],
top_logprobs=[{7: Logprob(float("nan"), rank=0), 8: Logprob(-0.2, rank=1)}],
)
entry = result.content[0]
assert entry.rank is None
assert entry.top_logprobs[0].rank is None
# Clamped like the Rust frontend (max(nan, x) is nan in Python).
assert entry.logprob == -9999.0
assert entry.top_logprobs[0].logprob == -9999.0
json.dumps(result.model_dump(), allow_nan=False)
def test_text_logprobs_clamp_a_nan_logprob():
result = ServingTokens._create_text_logprobs(
None,
token_ids=[7],
top_logprobs=[{7: Logprob(float("nan"), rank=0, decoded_token="a")}],
tokenizer=MagicMock(),
)
entry = result.content[0]
assert entry.logprob == -9999.0
assert entry.top_logprobs[0].logprob == -9999.0
json.dumps(result.model_dump(), allow_nan=False)
tests/entrypoints/serve/exception_handling/test_error_sanitization.py
+1 -1
View File
@@ -120,7 +120,7 @@ class TestAffectedModulesUseSanitize:
@pytest.mark.parametrize(
"module",
[
"vllm.entrypoints.speech_to_text.realtime.connection",
"vllm.entrypoints.serve.exception_handling.handlers.http",
],
)
def test_module_calls_sanitize_message(self, module: str):
tests/entrypoints/serve/lora/__init__.py
View File
tests/entrypoints/serve/lora/test_lora_adapters.py
-371
View File
@@ -1,371 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
import json
import shutil
from contextlib import suppress
import openai # use the official client for correctness check
import pytest
import pytest_asyncio
from tests.utils import RemoteOpenAIServer
# any model with a chat template should work here
MODEL_NAME = "Qwen/Qwen3-0.6B"
BADREQUEST_CASES = [
(
"test_rank",
{"r": 1024},
"is greater than max_lora_rank",
),
("test_dora", {"use_dora": True}, "does not yet support DoRA"),
("test_pissa", {"init_lora_weights": "pissa"}, "init_lora_weights='pissa'"),
(
"test_modules_to_save",
{"modules_to_save": ["lm_head"]},
"Unsupported modules_to_save",
),
]
@pytest.fixture(scope="module", params=[True])
def server_with_lora_modules_json(request, qwen3_lora_files):
# Define the json format LoRA module configurations
lora_module_1 = {
"name": "qwen3-lora",
"path": qwen3_lora_files,
"base_model_name": MODEL_NAME,
}
args = [
# use half precision for speed and memory savings in CI environment
"--dtype",
"bfloat16",
"--max-model-len",
"8192",
"--enforce-eager",
# lora config below
"--enable-lora",
"--lora-modules",
json.dumps(lora_module_1),
"--max-lora-rank",
"64",
"--max-cpu-loras",
"2",
"--max-num-seqs",
"64",
]
# Enable the /v1/load_lora_adapter endpoint
envs = {"VLLM_ALLOW_RUNTIME_LORA_UPDATING": "True"}
with RemoteOpenAIServer(MODEL_NAME, args, env_dict=envs) as remote_server:
yield remote_server
@pytest_asyncio.fixture
async def client(server_with_lora_modules_json):
async with server_with_lora_modules_json.get_async_client() as async_client:
yield async_client
@pytest.mark.asyncio
async def test_static_lora_lineage(client: openai.AsyncOpenAI, qwen3_lora_files):
models = await client.models.list()
models = models.data
served_model = models[0]
lora_models = models[1:]
assert served_model.id == MODEL_NAME
assert served_model.root == MODEL_NAME
assert served_model.parent is None
assert all(lora_model.root == qwen3_lora_files for lora_model in lora_models)
assert all(lora_model.parent == MODEL_NAME for lora_model in lora_models)
assert lora_models[0].id == "qwen3-lora"
@pytest.mark.asyncio
async def test_dynamic_lora_lineage(client: openai.AsyncOpenAI, qwen3_lora_files):
response = await client.post(
"load_lora_adapter",
cast_to=str,
body={"lora_name": "qwen3-lora-3", "lora_path": qwen3_lora_files},
)
# Ensure adapter loads before querying /models
assert "success" in response
models = await client.models.list()
models = models.data
dynamic_lora_model = models[-1]
assert dynamic_lora_model.root == qwen3_lora_files
assert dynamic_lora_model.parent == MODEL_NAME
assert dynamic_lora_model.id == "qwen3-lora-3"
@pytest.mark.asyncio
async def test_load_lora_adapter_with_same_name_replaces_inplace(
client: openai.AsyncOpenAI, qwen3_meowing_lora_files, qwen3_woofing_lora_files
):
"""Test that loading a LoRA adapter with the same name replaces it inplace."""
adapter_name = "replaceable-adapter"
messages = [
{"content": "Follow the instructions to make animal noises", "role": "system"},
{"content": "Make your favorite animal noise.", "role": "user"},
]
# Load LoRA that makes model meow
response = await client.post(
"load_lora_adapter",
cast_to=str,
body={"lora_name": adapter_name, "lora_path": qwen3_meowing_lora_files},
)
assert "success" in response.lower()
completion = await client.chat.completions.create(
model=adapter_name,
messages=messages,
max_tokens=10,
)
assert "Meow Meow Meow" in completion.choices[0].message.content
# Load LoRA that makes model woof
response = await client.post(
"load_lora_adapter",
cast_to=str,
body={
"lora_name": adapter_name,
"lora_path": qwen3_woofing_lora_files,
"load_inplace": True,
},
)
assert "success" in response.lower()
completion = await client.chat.completions.create(
model=adapter_name,
messages=messages,
max_tokens=10,
)
assert "Woof Woof Woof" in completion.choices[0].message.content
@pytest.mark.asyncio
async def test_load_lora_adapter_with_load_inplace_false_errors(
client: openai.AsyncOpenAI, qwen3_meowing_lora_files
):
"""Test that load_inplace=False returns an error when adapter already exists."""
adapter_name = "test-load-inplace-false"
# Load LoRA adapter first time (should succeed)
response = await client.post(
"load_lora_adapter",
cast_to=str,
body={"lora_name": adapter_name, "lora_path": qwen3_meowing_lora_files},
)
assert "success" in response.lower()
# Try to load the same adapter again with load_inplace=False (should fail)
with pytest.raises(openai.BadRequestError) as exc_info:
await client.post(
"load_lora_adapter",
cast_to=str,
body={
"lora_name": adapter_name,
"lora_path": qwen3_meowing_lora_files,
},
)
# Verify the error message
assert "already been loaded" in str(exc_info.value)
@pytest.mark.asyncio
async def test_dynamic_lora_not_found(client: openai.AsyncOpenAI):
with pytest.raises(openai.NotFoundError):
await client.post(
"load_lora_adapter",
cast_to=str,
body={"lora_name": "notfound", "lora_path": "/not/an/adapter"},
)
@pytest.mark.asyncio
async def test_dynamic_lora_invalid_files(client: openai.AsyncOpenAI, tmp_path):
invalid_files = tmp_path / "invalid_files"
invalid_files.mkdir()
(invalid_files / "adapter_config.json").write_text("this is not json")
with pytest.raises(openai.InternalServerError):
await client.post(
"load_lora_adapter",
cast_to=str,
body={"lora_name": "invalid-json", "lora_path": str(invalid_files)},
)
@pytest.mark.asyncio
@pytest.mark.parametrize("test_name,config_change,expected_error", BADREQUEST_CASES)
async def test_dynamic_lora_badrequests(
client: openai.AsyncOpenAI,
tmp_path,
qwen3_lora_files,
test_name: str,
config_change: dict,
expected_error: str,
):
# Create test directory
test_dir = tmp_path / test_name
# Copy adapter files
shutil.copytree(qwen3_lora_files, test_dir)
# Load and modify configuration
config_path = test_dir / "adapter_config.json"
with open(config_path) as f:
adapter_config = json.load(f)
# Apply configuration changes
adapter_config.update(config_change)
# Save modified configuration
with open(config_path, "w") as f:
json.dump(adapter_config, f)
# Test loading the adapter
with pytest.raises(openai.InternalServerError, match=expected_error):
await client.post(
"load_lora_adapter",
cast_to=str,
body={"lora_name": test_name, "lora_path": str(test_dir)},
)
@pytest.mark.asyncio
async def test_multiple_lora_adapters(
client: openai.AsyncOpenAI, tmp_path, qwen3_lora_files
):
"""Validate that many loras can be dynamically registered and inferenced
with concurrently"""
# This test file configures the server with --max-cpu-loras=2 and this test
# will concurrently load 10 adapters, so it should flex the LRU cache
async def load_and_run_adapter(adapter_name: str):
await client.post(
"load_lora_adapter",
cast_to=str,
body={"lora_name": adapter_name, "lora_path": str(qwen3_lora_files)},
)
for _ in range(3):
await client.completions.create(
model=adapter_name,
prompt=["Hello there", "Foo bar bazz buzz"],
max_tokens=5,
)
lora_tasks = []
for i in range(10):
lora_tasks.append(asyncio.create_task(load_and_run_adapter(f"adapter_{i}")))
results, _ = await asyncio.wait(lora_tasks)
for r in results:
assert not isinstance(r, Exception), f"Got exception {r}"
@pytest.mark.asyncio
async def test_loading_invalid_adapters_does_not_break_others(
client: openai.AsyncOpenAI, tmp_path, qwen3_lora_files
):
invalid_files = tmp_path / "invalid_files"
invalid_files.mkdir()
(invalid_files / "adapter_config.json").write_text("this is not json")
stop_good_requests_event = asyncio.Event()
async def run_good_requests(client):
# Run chat completions requests until event set
results = []
while not stop_good_requests_event.is_set():
try:
batch = await client.completions.create(
model="qwen3-lora",
prompt=["Hello there", "Foo bar bazz buzz"],
max_tokens=5,
)
results.append(batch)
except Exception as e:
results.append(e)
return results
# Create task to run good requests
good_task = asyncio.create_task(run_good_requests(client))
# Run a bunch of bad adapter loads
for _ in range(25):
with suppress(openai.NotFoundError):
await client.post(
"load_lora_adapter",
cast_to=str,
body={"lora_name": "notfound", "lora_path": "/not/an/adapter"},
)
for _ in range(25):
with suppress(openai.InternalServerError):
await client.post(
"load_lora_adapter",
cast_to=str,
body={"lora_name": "invalid", "lora_path": str(invalid_files)},
)
# Ensure all the running requests with lora adapters succeeded
stop_good_requests_event.set()
results = await good_task
for r in results:
assert not isinstance(r, Exception), f"Got exception {r}"
# Ensure we can load another adapter and run it
await client.post(
"load_lora_adapter",
cast_to=str,
body={"lora_name": "valid", "lora_path": qwen3_lora_files},
)
await client.completions.create(
model="valid",
prompt=["Hello there", "Foo bar bazz buzz"],
max_tokens=5,
)
@pytest.mark.asyncio
async def test_beam_search_with_lora_adapters(
client: openai.AsyncOpenAI,
tmp_path,
qwen3_lora_files,
):
"""Validate that async beam search can be used with lora."""
async def load_and_run_adapter(adapter_name: str):
await client.post(
"load_lora_adapter",
cast_to=str,
body={"lora_name": adapter_name, "lora_path": str(qwen3_lora_files)},
)
for _ in range(3):
await client.completions.create(
model=adapter_name,
prompt=["Hello there", "Foo bar bazz buzz"],
max_tokens=5,
extra_body=dict(use_beam_search=True),
)
lora_tasks = []
for i in range(3):
lora_tasks.append(asyncio.create_task(load_and_run_adapter(f"adapter_{i}")))
results, _ = await asyncio.wait(lora_tasks)
for r in results:
assert not isinstance(r, Exception), f"Got exception {r}"
tests/entrypoints/serve/utils/test_request_logger.py
-1
View File
@@ -214,7 +214,6 @@ def test_request_logger_log_outputs_integration():
prompt_token_ids=[1, 2, 3],
prompt_embeds=None,
params=None,
lora_request=None,
)
request_logger.log_outputs(
tests/entrypoints/speech_to_text/__init__.py
View File
tests/entrypoints/speech_to_text/conftest.py
-39
View File
@@ -1,39 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm.assets.audio import AudioAsset
def add_attention_backend(server_args, attention_config):
"""Append attention backend CLI arg if specified.
Args:
server_args: List of server arguments to extend in-place.
attention_config: Dict with 'backend' key, or None.
"""
if attention_config and "backend" in attention_config:
server_args.extend(["--attention-backend", attention_config["backend"]])
@pytest.fixture
def mary_had_lamb():
path = AudioAsset("mary_had_lamb").get_local_path()
with open(str(path), "rb") as f:
yield f
@pytest.fixture
def winning_call():
path = AudioAsset("winning_call").get_local_path()
with open(str(path), "rb") as f:
yield f
@pytest.fixture
def foscolo():
# Test translation it->en
path = AudioAsset("azacinto_foscolo").get_local_path()
with open(str(path), "rb") as f:
yield f
tests/entrypoints/speech_to_text/transcription/__init__.py
View File
tests/entrypoints/speech_to_text/transcription/test_enable_force_include_usage.py
-61
View File
@@ -1,61 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import pytest_asyncio
from tests.utils import RemoteOpenAIServer
@pytest.fixture(scope="module")
def transcription_server_with_force_include_usage():
args = [
# use half precision for speed and memory savings in CI environment
"--dtype",
"bfloat16",
"--max-num-seqs",
"4",
"--enforce-eager",
"--enable-force-include-usage",
"--gpu-memory-utilization",
"0.2",
]
with RemoteOpenAIServer("openai/whisper-large-v3-turbo", args) as remote_server:
yield remote_server
@pytest_asyncio.fixture
async def transcription_client_with_force_include_usage(
transcription_server_with_force_include_usage,
):
async with (
transcription_server_with_force_include_usage.get_async_client() as async_client
):
yield async_client
@pytest.mark.asyncio
async def test_transcription_with_enable_force_include_usage(
transcription_client_with_force_include_usage, winning_call
):
res = (
await transcription_client_with_force_include_usage.audio.transcriptions.create(
model="openai/whisper-large-v3-turbo",
file=winning_call,
language="en",
temperature=0.0,
stream=True,
timeout=30,
)
)
async for chunk in res:
if not len(chunk.choices):
# final usage sent
usage = chunk.usage
assert isinstance(usage, dict)
assert usage["prompt_tokens"] > 0
assert usage["completion_tokens"] > 0
assert usage["total_tokens"] > 0
else:
assert not hasattr(chunk, "usage")
tests/kernels/moe/test_moe_weight_loading_padded.py
+13 -17
View File
@@ -592,13 +592,12 @@ class TestLoadWeightsExpertMapping:
return calls
@pytest.mark.parametrize(
"projs,lora_prefix",
"projs",
[
(("gate_proj", "down_proj", "up_proj"), ""),
(("w1", "w2", "w3"), ""),
(("up_proj", "down_proj", "up_proj"), ""),
(("up_proj", "down_proj", None), ""),
(("gate_proj", "down_proj", "up_proj"), "base_layer."),
("gate_proj", "down_proj", "up_proj"),
("w1", "w2", "w3"),
("up_proj", "down_proj", "up_proj"),
("up_proj", "down_proj", None),
],
)
@pytest.mark.parametrize(
@@ -610,12 +609,11 @@ class TestLoadWeightsExpertMapping:
],
)
def test_per_expert_weights_keep_all_physical_destinations(
self, projs, lora_prefix, suffix, shape, dtype
self, projs, suffix, shape, dtype
):
mapping = self._mapping(
projs,
num_redundant_experts=4,
lora_base_layer_prefix=lora_prefix,
is_gated=projs[2] is not None,
)
for logical_id, physical_ids in [(1, (1, 17)), (10, (10,))]:
@@ -623,7 +621,7 @@ class TestLoadWeightsExpertMapping:
if not proj:
continue
weight = torch.full(shape, logical_id + 1, dtype=dtype)
name = f"{logical_id}.{proj}.{lora_prefix}{suffix}"
name = f"{logical_id}.{proj}.{suffix}"
calls = self._load(mapping, [(name, weight)], suffix)
shards = [s for p, s in zip(projs, ("w1", "w2", "w3")) if p == proj]
expected = [
@@ -636,20 +634,18 @@ class TestLoadWeightsExpertMapping:
torch.testing.assert_close(call[3], weight, rtol=0, atol=0)
@pytest.mark.parametrize(
"projs,fused_name,lora_prefix",
"projs,fused_name",
[
(("gate_proj", "down_proj", "up_proj"), "gate_up_proj", ""),
(("w1", "w2", "w3"), "w13", "base_layer."),
(("gate_proj", "down_proj", "up_proj"), "gate_up_proj"),
(("w1", "w2", "w3"), "w13"),
],
)
def test_per_expert_fused_gate_up_keeps_both_halves(
self, projs, fused_name, lora_prefix
self, projs, fused_name
):
mapping = self._mapping(
projs, num_redundant_experts=4, lora_base_layer_prefix=lora_prefix
)
mapping = self._mapping(projs, num_redundant_experts=4)
weight = torch.arange(12).reshape(4, 3)
calls = self._load(mapping, [(f"1.{fused_name}.{lora_prefix}weight", weight)])
calls = self._load(mapping, [(f"1.{fused_name}.weight", weight)])
assert [call[:3] for call in calls] == [
("w13_weight", shard, expert)
for expert in (1, 17)
tests/lora/__init__.py
View File
tests/lora/conftest.py
-357
View File
@@ -1,357 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import tempfile
from collections import OrderedDict
from importlib import reload
from unittest.mock import MagicMock
import pytest
import torch
import torch.nn as nn
from vllm.distributed import (
cleanup_dist_env_and_memory,
init_distributed_environment,
initialize_model_parallel,
)
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
MergedColumnParallelLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from vllm.model_executor.models.interfaces import SupportsLoRA
from vllm.platforms import current_platform
from vllm.transformers_utils.repo_utils import hf_api
@pytest.fixture()
def should_do_global_cleanup_after_test(request) -> bool:
"""Allow subdirectories to skip global cleanup by overriding this fixture.
This can provide a ~10x speedup for non-GPU unit tests since they don't need
to initialize torch.
"""
return not request.node.get_closest_marker("skip_global_cleanup")
@pytest.fixture(autouse=True)
def cleanup_fixture(should_do_global_cleanup_after_test: bool):
yield
if should_do_global_cleanup_after_test:
cleanup_dist_env_and_memory(shutdown_ray=True)
@pytest.fixture
def maybe_enable_lora_dual_stream(monkeypatch: pytest.MonkeyPatch):
if current_platform.is_cuda():
monkeypatch.setenv("VLLM_LORA_ENABLE_DUAL_STREAM", "1")
import vllm.lora.layers.base_linear
if not hasattr(vllm.lora.layers.base_linear, "lora_linear_async"):
# Reload the module to ensure the environment variable takes effect.
reload(vllm.lora.layers.base_linear)
yield
@pytest.fixture
def dist_init():
from tests.utils import ensure_current_vllm_config
temp_file = tempfile.mkstemp()[1]
backend = "gloo" if current_platform.is_tpu() else current_platform.dist_backend
with ensure_current_vllm_config():
init_distributed_environment(
world_size=1,
rank=0,
distributed_init_method=f"file://{temp_file}",
local_rank=0,
backend=backend,
)
initialize_model_parallel(1, 1)
yield
cleanup_dist_env_and_memory(shutdown_ray=True)
class DummyLoRAModel(nn.Sequential, SupportsLoRA):
pass
@pytest.fixture
def dummy_model(default_vllm_config) -> nn.Module:
model = DummyLoRAModel(
OrderedDict(
[
("dense1", ColumnParallelLinear(764, 100)),
("dense2", RowParallelLinear(100, 50)),
(
"layer1",
nn.Sequential(
OrderedDict(
[
("dense1", ColumnParallelLinear(100, 10)),
("dense2", RowParallelLinear(10, 50)),
]
)
),
),
("act2", nn.ReLU()),
("output", ColumnParallelLinear(50, 10)),
("outact", nn.Sigmoid()),
# Special handling for lm_head & sampler
("lm_head", ParallelLMHead(32064, 10)),
("logits_processor", LogitsProcessor(32064)),
]
)
)
model.config = MagicMock()
model.embedding_modules = {"lm_head": "lm_head"}
model.unpadded_vocab_size = 32064
return model
@pytest.fixture
def dummy_model_gate_up(default_vllm_config) -> nn.Module:
model = DummyLoRAModel(
OrderedDict(
[
("dense1", ColumnParallelLinear(764, 100)),
("dense2", RowParallelLinear(100, 50)),
(
"layer1",
nn.Sequential(
OrderedDict(
[
("dense1", ColumnParallelLinear(100, 10)),
("dense2", RowParallelLinear(10, 50)),
]
)
),
),
("act2", nn.ReLU()),
("gate_up_proj", MergedColumnParallelLinear(50, [5, 5])),
("outact", nn.Sigmoid()),
# Special handling for lm_head & sampler
("lm_head", ParallelLMHead(32064, 10)),
("logits_processor", LogitsProcessor(32064)),
]
)
)
model.config = MagicMock()
model.packed_modules_mapping = {
"gate_up_proj": [
"gate_proj",
"up_proj",
],
}
model.embedding_modules = {"lm_head": "lm_head"}
model.unpadded_vocab_size = 32064
return model
@pytest.fixture(scope="session")
def mixtral_lora_files():
# Note: this module has incorrect adapter_config.json to test
# https://github.com/vllm-project/vllm/pull/5909/files.
return hf_api().snapshot_download(repo_id="SangBinCho/mixtral-lora")
@pytest.fixture(scope="session")
def chatglm3_lora_files():
return hf_api().snapshot_download(repo_id="jeeejeee/chatglm3-text2sql-spider")
@pytest.fixture(scope="session")
def baichuan_lora_files():
return hf_api().snapshot_download(repo_id="jeeejeee/baichuan7b-text2sql-spider")
@pytest.fixture(scope="session")
def baichuan_zero_lora_files():
# all the lora_B weights are initialized to zero.
return hf_api().snapshot_download(repo_id="jeeejeee/baichuan7b-zero-init")
@pytest.fixture(scope="session")
def baichuan_regex_lora_files():
return hf_api().snapshot_download(repo_id="jeeejeee/baichuan-7b-lora-zero-regex")
@pytest.fixture(scope="session")
def ilama_lora_files():
return hf_api().snapshot_download(repo_id="jeeejeee/ilama-text2sql-spider")
@pytest.fixture(scope="session")
def minicpmv_lora_files():
return hf_api().snapshot_download(repo_id="jeeejeee/minicpmv25-lora-pokemon")
@pytest.fixture(scope="session")
def qwen2vl_lora_files():
return hf_api().snapshot_download(repo_id="jeeejeee/qwen2-vl-lora-pokemon")
@pytest.fixture(scope="session")
def qwen25vl_lora_files():
return hf_api().snapshot_download(repo_id="jeeejeee/qwen25-vl-lora-pokemon")
@pytest.fixture(scope="session")
def qwen2vl_language_lora_files():
return hf_api().snapshot_download(
repo_id="prashanth058/qwen2vl-flickr-lora-language"
)
@pytest.fixture(scope="session")
def qwen2vl_vision_tower_connector_lora_files():
return hf_api().snapshot_download(
repo_id="prashanth058/qwen2vl-flickr-lora-tower-connector"
)
@pytest.fixture(scope="session")
def qwen2vl_vision_tower_lora_files():
return hf_api().snapshot_download(repo_id="prashanth058/qwen2vl-flickr-lora-tower")
@pytest.fixture(scope="session")
def qwen25vl_vision_lora_files():
return hf_api().snapshot_download(
repo_id="EpochEcho/qwen2.5-3b-vl-lora-vision-connector"
)
@pytest.fixture(scope="session")
def qwen3vl_vision_lora_files():
return hf_api().snapshot_download(
repo_id="EpochEcho/qwen3-4b-vl-lora-vision-connector"
)
@pytest.fixture(scope="session")
def gemma4_vision_lora_files():
return hf_api().snapshot_download(repo_id="EpochEcho/gemma4-e2b-it-lora-pokemon")
@pytest.fixture(scope="session")
def qwen3_meowing_lora_files():
"""Download Qwen3 Meow LoRA files once per test session."""
return hf_api().snapshot_download(repo_id="Jackmin108/Qwen3-0.6B-Meow-LoRA")
@pytest.fixture(scope="session")
def qwen3_woofing_lora_files():
"""Download Qwen3 Woof LoRA files once per test session."""
return hf_api().snapshot_download(repo_id="Jackmin108/Qwen3-0.6B-Woof-LoRA")
@pytest.fixture(scope="session")
def qwen3_guard_star_trek_lora_files():
return hf_api().snapshot_download(
repo_id="geoffmunn/Qwen3Guard-StarTrek-Classification-0.6B",
allow_patterns=["adapter_config.json", "adapter_model.safetensors"],
)
@pytest.fixture(scope="session")
def qwen3_guard_new_zealand_lora_files():
return hf_api().snapshot_download(
repo_id="geoffmunn/Qwen3Guard-NewZealand-Classification-0.6B",
allow_patterns=["adapter_config.json", "adapter_model.safetensors"],
)
@pytest.fixture(scope="session")
def skywork_qwen3_reward_lora_files():
return hf_api().snapshot_download(
repo_id="AmirMohseni/skywork-qwen3-0.6b-reward-lora",
allow_patterns=["adapter_config.json", "adapter_model.safetensors"],
)
@pytest.fixture(scope="session")
def deepseekv2_lora_files():
return hf_api().snapshot_download(repo_id="wuchen01/DeepSeek-V2-Lite-Chat-All-LoRA")
@pytest.fixture(scope="session")
def gptoss20b_lora_files():
return hf_api().snapshot_download(
repo_id="jeeejeee/gpt-oss-20b-lora-adapter-text2sql"
)
@pytest.fixture(scope="session")
def qwen3moe_lora_files():
return hf_api().snapshot_download(repo_id="jeeejeee/qwen3-moe-text2sql-spider")
@pytest.fixture(scope="session")
def olmoe_lora_files():
return hf_api().snapshot_download(repo_id="jeeejeee/olmoe-instruct-text2sql-spider")
@pytest.fixture(scope="session")
def qwen3_lora_files():
return hf_api().snapshot_download(repo_id="charent/self_cognition_Alice")
@pytest.fixture(scope="session")
def llama32_lora_huggingface_id():
# huggingface repo id is used to test lora runtime downloading.
return "jeeejeee/llama32-3b-text2sql-spider"
@pytest.fixture(scope="session")
def llama32_lora_files(llama32_lora_huggingface_id):
return hf_api().snapshot_download(repo_id=llama32_lora_huggingface_id)
@pytest.fixture(scope="session")
def whisper_lora_files():
return hf_api().snapshot_download(
repo_id="chengyili2005/whisper-small-mandarin-lora"
)
@pytest.fixture(scope="session")
def qwen35_text_lora_files():
return hf_api().snapshot_download(repo_id="jeeejeee/qwen35-4b-text-only-sql-lora")
@pytest.fixture(scope="session")
def qwen35_vl_lora_files():
return hf_api().snapshot_download(
repo_id="jeeejeee/qwen35-4b-all-linear-pokemon-lora"
)
@pytest.fixture(scope="session")
def qwen36_moe_2d_lora_files():
return hf_api().snapshot_download(
repo_id="jeeejeee/qwen36-35ba3b-2d-weights-poken-lora"
)
@pytest.fixture(scope="session")
def qwen36_moe_3d_lora_files():
return hf_api().snapshot_download(
repo_id="jeeejeee/qwen36-35ba3b-moe-all-linear-poken-lora"
)
@pytest.fixture
def reset_default_device():
"""Some tests, such as `test_punica_ops.py`, explicitly set the
default device, which can affect subsequent tests. Adding this fixture
helps avoid this problem.
"""
original_device = torch.get_default_device()
yield
torch.set_default_device(original_device)
tests/lora/test_layers_utils.py
-39
View File
@@ -1,39 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from torch import nn
from vllm.lora.layers.utils import _get_lora_device
pytestmark = pytest.mark.skip_global_cleanup
def _param() -> nn.Parameter:
return nn.Parameter(torch.empty(1), requires_grad=False)
def test_get_lora_device_unquantized():
base_layer = nn.Module()
base_layer.weight = _param()
assert _get_lora_device(base_layer) == base_layer.weight.device
def test_get_lora_device_gptq_awq():
base_layer = nn.Module()
base_layer.qweight = _param()
assert _get_lora_device(base_layer) == base_layer.qweight.device
def test_get_lora_device_ark_linear():
base_layer = nn.Module()
base_layer.ark_linear = nn.Module()
base_layer.ark_linear.qweight = _param()
assert _get_lora_device(base_layer) == base_layer.ark_linear.qweight.device
def test_get_lora_device_unsupported_raises():
base_layer = nn.Module()
with pytest.raises(ValueError, match="Unsupported base layer"):
_get_lora_device(base_layer)
tests/lora/test_lora_functions.py
-114
View File
@@ -1,114 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Script to test add_lora, remove_lora, pin_lora, list_loras functions."""
import pytest
from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs
from vllm.entrypoints.launchers.api_server.entry import (
build_async_engine_client_from_engine_args,
)
from vllm.lora.request import LoRARequest
from vllm.v1.engine.llm_engine import LLMEngine
MODEL_PATH = "Qwen/Qwen3-0.6B"
LORA_MODULE_PATH = "charent/self_cognition_Alice"
LORA_RANK = 8
def make_lora_request(lora_id: int):
return LoRARequest(
lora_name=f"{lora_id}", lora_int_id=lora_id, lora_path=LORA_MODULE_PATH
)
def test_lora_functions_sync():
max_loras = 4
# Create engine in eager-mode. Due to high max_loras, the CI can
# OOM during cuda-graph capture.
engine_args = EngineArgs(
model=MODEL_PATH,
enable_lora=True,
max_loras=max_loras,
max_lora_rank=LORA_RANK,
max_model_len=128,
gpu_memory_utilization=0.8,
enforce_eager=True,
)
llm = LLMEngine.from_engine_args(engine_args)
def run_check(fn, args, expected: list):
fn(args)
assert set(llm.list_loras()) == set(expected)
run_check(llm.add_lora, make_lora_request(1), [1])
run_check(llm.add_lora, make_lora_request(2), [1, 2])
# Pin LoRA 1 and test that it is never removed on subsequent adds.
run_check(llm.pin_lora, 1, [1, 2])
run_check(llm.add_lora, make_lora_request(3), [1, 2, 3])
run_check(llm.add_lora, make_lora_request(4), [1, 2, 3, 4])
run_check(llm.add_lora, make_lora_request(5), [1, 5, 3, 4])
run_check(llm.add_lora, make_lora_request(6), [1, 5, 6, 4])
run_check(llm.add_lora, make_lora_request(7), [1, 5, 6, 7])
run_check(llm.add_lora, make_lora_request(8), [1, 8, 6, 7])
run_check(llm.add_lora, make_lora_request(9), [1, 8, 9, 7])
run_check(llm.add_lora, make_lora_request(10), [1, 8, 9, 10])
# Remove LoRA 1 and continue adding.
run_check(llm.remove_lora, 1, [8, 9, 10])
run_check(llm.add_lora, make_lora_request(11), [8, 9, 10, 11])
run_check(llm.add_lora, make_lora_request(12), [12, 9, 10, 11])
run_check(llm.add_lora, make_lora_request(13), [12, 13, 10, 11])
# Remove all LoRAs.
run_check(llm.remove_lora, 13, [12, 10, 11])
run_check(llm.remove_lora, 12, [10, 11])
run_check(llm.remove_lora, 11, [10])
run_check(llm.remove_lora, 10, [])
@pytest.mark.asyncio
async def test_lora_functions_async():
max_loras = 4
engine_args = AsyncEngineArgs(
model=MODEL_PATH,
enable_lora=True,
max_loras=max_loras,
max_lora_rank=LORA_RANK,
max_model_len=128,
gpu_memory_utilization=0.8,
enforce_eager=True,
)
async def run_check(fn, args, expected: list):
await fn(args)
assert set(await llm.list_loras()) == set(expected)
async with build_async_engine_client_from_engine_args(engine_args) as llm:
await run_check(llm.add_lora, make_lora_request(1), [1])
await run_check(llm.add_lora, make_lora_request(2), [1, 2])
# Pin LoRA 1 and test that it is never removed on subsequent adds.
await run_check(llm.pin_lora, 1, [1, 2])
await run_check(llm.add_lora, make_lora_request(3), [1, 2, 3])
await run_check(llm.add_lora, make_lora_request(4), [1, 2, 3, 4])
await run_check(llm.add_lora, make_lora_request(5), [1, 5, 3, 4])
await run_check(llm.add_lora, make_lora_request(6), [1, 5, 6, 4])
await run_check(llm.add_lora, make_lora_request(7), [1, 5, 6, 7])
await run_check(llm.add_lora, make_lora_request(8), [1, 8, 6, 7])
await run_check(llm.add_lora, make_lora_request(9), [1, 8, 9, 7])
await run_check(llm.add_lora, make_lora_request(10), [1, 8, 9, 10])
# Remove LoRA 1 and continue adding.
await run_check(llm.remove_lora, 1, [8, 9, 10])
await run_check(llm.add_lora, make_lora_request(11), [8, 9, 10, 11])
await run_check(llm.add_lora, make_lora_request(12), [12, 9, 10, 11])
await run_check(llm.add_lora, make_lora_request(13), [12, 13, 10, 11])
# Remove all LoRAs
await run_check(llm.remove_lora, 13, [12, 10, 11])
await run_check(llm.remove_lora, 12, [10, 11])
await run_check(llm.remove_lora, 11, [10])
await run_check(llm.remove_lora, 10, [])
tests/lora/test_lora_utils.py
-51
View File
@@ -1,51 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm.lora.utils import is_in_target_modules
class TestIsInTargetModules:
"""Tests for is_in_target_modules (deployment-time filter)."""
def test_none_allows_all(self):
assert is_in_target_modules("model.layers.0.self_attn.o_proj", None)
def test_suffix_in_target(self):
assert is_in_target_modules(
"model.layers.0.self_attn.o_proj", ["o_proj", "q_proj"]
)
def test_suffix_not_in_target(self):
assert not is_in_target_modules(
"model.layers.0.self_attn.o_proj", ["q_proj", "k_proj"]
)
def test_empty_target_modules(self):
assert not is_in_target_modules("model.layers.0.self_attn.o_proj", [])
def test_exact_name_match(self):
assert is_in_target_modules("dense1", ["dense1", "dense2"])
def test_exact_name_no_match(self):
assert not is_in_target_modules("dense3", ["dense1", "dense2"])
def test_packed_parent_matches_child_target_modules(self):
assert is_in_target_modules(
"model.layers.0.mlp.gate_up_proj",
["gate_proj", "up_proj"],
{"gate_up_proj": ["gate_proj", "up_proj"]},
)
def test_packed_child_matches_parent_target_modules(self):
assert is_in_target_modules(
"model.layers.0.mlp.gate_proj",
["gate_up_proj"],
{"gate_up_proj": ["gate_proj", "up_proj"]},
)
def test_fused_parent_matches_child_target_modules(self):
assert is_in_target_modules(
"model.layers.0.self_attn.fused_qkv_a_proj",
["q_a_proj", "kv_a_proj_with_mqa"],
{"fused_qkv_a_proj": ["q_a_proj", "kv_a_proj_with_mqa"]},
)
tests/lora/test_lora_weights.py
-109
View File
@@ -1,109 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from vllm.lora.lora_weights import LoRALayerWeights, PackedLoRALayerWeights
from vllm.lora.peft_helper import PEFTHelper
RANK, ALPHA = 16, 32
# PEFTHelper stores alpha/sqrt(r) for use_rslora adapters.
RSLORA_SCALING = ALPHA / RANK**0.5
@pytest.fixture(autouse=True)
def should_do_global_cleanup_after_test():
# Pure-python weight packing; skip GPU/ray cleanup.
return False
def moe_loras(scaling: float, stacked: bool = False) -> list[LoRALayerWeights]:
a_shape = (3, RANK, 4) if stacked else (RANK, 4)
b_shape = (3, 6, RANK) if stacked else (6, RANK)
return [
LoRALayerWeights(
module_name=f"experts.w{i}",
rank=RANK,
lora_alpha=ALPHA,
lora_a=torch.randn(*a_shape),
lora_b=torch.randn(*b_shape),
scaling=scaling,
)
for i in (1, 2, 3)
]
@pytest.mark.parametrize(
"packer,stacked",
[
(PackedLoRALayerWeights.pack_moe, False),
(PackedLoRALayerWeights.pack_moe_stacked, True),
],
)
def test_moe_packing_uses_stored_scaling(packer, stacked: bool):
packed = packer(moe_loras(RSLORA_SCALING, stacked), "experts")
assert packed.scaling == pytest.approx([RSLORA_SCALING] * 3)
def test_non_gated_moe_keeps_w3_unscaled():
packed = PackedLoRALayerWeights.pack_moe(
moe_loras(RSLORA_SCALING), "experts", is_non_gated_moe=True
)
assert packed.scaling[:2] == pytest.approx([RSLORA_SCALING] * 2)
assert packed.scaling[2] == 1.0
@pytest.mark.parametrize(
"packer,stacked",
[
(PackedLoRALayerWeights.pack_moe, False),
(PackedLoRALayerWeights.pack_moe_stacked, True),
],
)
def test_moe_packing_keeps_per_projection_scaling(packer, stacked: bool):
# e.g. rank_pattern / alpha_pattern giving gate, down and up different scalings
loras = moe_loras(1.0, stacked)
for lora, scaling in zip(loras, (2.0, 4.0, 8.0)):
lora.scaling = scaling
packed = packer(loras, "experts")
assert packed.scaling == pytest.approx([2.0, 4.0, 8.0])
def test_moe_packing_keeps_per_expert_scaling():
"""Experts of one projection can have different scaling, e.g. from
alpha_pattern."""
loras = moe_loras(RSLORA_SCALING) + moe_loras(RSLORA_SCALING)
loras[4].scaling = 2 * RSLORA_SCALING # w2 of expert 1
expected = [lora.lora_b * lora.scaling for lora in loras]
packed = PackedLoRALayerWeights.pack_moe(loras, "experts").optimize()
for i, lora_b in enumerate(packed.lora_b):
torch.testing.assert_close(lora_b, torch.stack(expected[i::3]))
def test_moe_packing_rejects_per_expert_rank():
"""Experts of one projection must share a rank to be stacked."""
loras = moe_loras(1.0) + moe_loras(1.0)
loras[4].lora_a = torch.randn(RANK // 2, 4)
loras[4].lora_b = torch.randn(6, RANK // 2)
with pytest.raises(ValueError, match="different LoRA ranks"):
PackedLoRALayerWeights.pack_moe(loras, "experts")
def test_from_config_uses_checkpoint_module_name():
peft_helper = PEFTHelper(
r=RANK,
lora_alpha=ALPHA,
target_modules=["q_proj"],
rank_pattern={r"^model\.layers\.0\.self_attn\.q_proj": 4},
)
# the pattern refers to the checkpoint name, not the name after weights mapping
lora = LoRALayerWeights.from_config(
"language_model.model.layers.0.self_attn.q_proj",
peft_helper,
"model.layers.0.self_attn.q_proj",
)
assert lora.rank == 4
assert lora.scaling == pytest.approx(ALPHA / 4)
lora = LoRALayerWeights.from_config("model.layers.1.self_attn.q_proj", peft_helper)
assert lora.rank == RANK
assert lora.scaling == pytest.approx(ALPHA / RANK)
tests/lora/test_moe_lora_ep_load.py
-89
View File
@@ -1,89 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""End-to-end correctness test for 2D MoE LoRA expert-parallel
load-time slicing
"""
import pytest
import torch
from vllm.lora.lora_model import LoRAModel, MoEEPLoadSpec
from vllm.lora.peft_helper import PEFTHelper
NUM_LAYERS = 48
GLOBAL_NUM_EXPERTS = 128
LOCAL_NUM_EXPERTS = 64 # ep_size = 2
EXPERT_PROJECTIONS = ("down_proj", "gate_proj", "up_proj")
NON_EXPERT_MODULES = ("q_proj", "k_proj", "v_proj", "o_proj", "gate")
def _expected_lora_modules() -> set[str]:
"""Replicate the set ``WorkerLoRAManager._load_adapter`` would build
from this model's ``packed_modules_mapping``."""
expected: set[str] = set(NON_EXPERT_MODULES)
for expert in range(GLOBAL_NUM_EXPERTS):
for proj in EXPERT_PROJECTIONS:
expected.add(f"experts.{expert}.{proj}")
return expected
def _load(lora_dir, peft_helper, *, moe_ep_spec, lora_id):
return LoRAModel.from_local_checkpoint(
lora_dir,
_expected_lora_modules(),
peft_helper=peft_helper,
lora_model_id=lora_id,
device="cpu",
moe_ep_spec=moe_ep_spec,
)
@pytest.mark.parametrize("ep_rank", [0, 1])
def test_moe_lora_ep2_real_qwen3moe(qwen3moe_lora_files, ep_rank):
"""ep_size=2 against the real Qwen3-MoE adapter: each rank's loaded
LoRA has the right size, the right expert membership, and the
right tensor values."""
peft_helper = PEFTHelper.from_local_dir(
qwen3moe_lora_files, max_position_embeddings=4096
)
# Baseline: no spec → loads every expert × projection × layer plus
# all non-expert LoRA modules.
ground_truth = _load(qwen3moe_lora_files, peft_helper, moe_ep_spec=None, lora_id=1)
expected_baseline = (
GLOBAL_NUM_EXPERTS * len(EXPERT_PROJECTIONS) * NUM_LAYERS
+ len(NON_EXPERT_MODULES) * NUM_LAYERS
)
assert len(ground_truth.loras) == expected_baseline
# Sliced load: only this rank's experts; non-expert LoRA is untouched.
spec = MoEEPLoadSpec(
ep_rank=ep_rank,
local_num_experts=LOCAL_NUM_EXPERTS,
global_num_experts=GLOBAL_NUM_EXPERTS,
)
sliced = _load(
qwen3moe_lora_files,
peft_helper,
moe_ep_spec=spec,
lora_id=100 + ep_rank,
)
expected_sliced = (
LOCAL_NUM_EXPERTS * len(EXPERT_PROJECTIONS) * NUM_LAYERS
+ len(NON_EXPERT_MODULES) * NUM_LAYERS
)
assert len(sliced.loras) == expected_sliced
expert_start = ep_rank * LOCAL_NUM_EXPERTS
expert_end = expert_start + LOCAL_NUM_EXPERTS
for name, lora in sliced.loras.items():
gt = ground_truth.loras[name]
torch.testing.assert_close(lora.lora_a, gt.lora_a)
torch.testing.assert_close(lora.lora_b, gt.lora_b)
if ".experts." in name:
expert_idx = int(name.split(".experts.")[-1].split(".")[0])
assert expert_start <= expert_idx < expert_end, (
f"non-local expert {expert_idx} leaked: {name}"
)
tests/lora/test_peft_helper.py
-260
View File
@@ -1,260 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import math
import shutil
import pytest
from vllm.config.lora import LoRAConfig
from vllm.lora.peft_helper import PEFTHelper
ERROR_CASES = [
(
"test_rank",
{"r": 1024},
"is greater than max_lora_rank",
),
("test_dora", {"use_dora": True}, "does not yet support DoRA"),
(
"test_modules_to_save",
{"modules_to_save": ["lm_head"]},
"Unsupported modules_to_save",
),
(
"test_rank_pattern",
{"r": 8, "rank_pattern": {"q_proj": 32}},
"LoRA rank 32 is greater than max_lora_rank",
),
("test_rank_zero", {"r": 0}, "must be a positive integer"),
("test_rank_negative", {"r": -8}, "must be a positive integer"),
("test_lora_bias", {"lora_bias": True}, "does not support LoRA bias"),
("test_pissa", {"init_lora_weights": "pissa"}, "init_lora_weights='pissa'"),
(
"test_pissa_niter",
{"init_lora_weights": "pissa_niter_4"},
"init_lora_weights='pissa_niter_4'",
),
("test_olora", {"init_lora_weights": "olora"}, "init_lora_weights='olora'"),
("test_corda", {"init_lora_weights": "corda"}, "init_lora_weights='corda'"),
("test_loftq", {"init_lora_weights": "loftq"}, "init_lora_weights='loftq'"),
(
"test_unknown_init",
{"init_lora_weights": "future_init"},
"init_lora_weights='future_init'",
),
("test_alora", {"alora_invocation_tokens": [1, 2]}, "Activated LoRA"),
("test_layer_replication", {"layer_replication": [[0, 2]]}, "layer_replication"),
("test_bdlora", {"use_bdlora": {"nblocks": 2}}, "BD-LoRA"),
("test_qalora", {"use_qalora": True}, "QALoRA"),
]
def test_peft_helper_pass(llama32_lora_files, tmp_path):
peft_helper = PEFTHelper.from_local_dir(
llama32_lora_files, max_position_embeddings=4096
)
lora_config = LoRAConfig(max_lora_rank=16, max_cpu_loras=3, max_loras=2)
peft_helper.validate_legal(lora_config)
assert peft_helper.r == 8
assert peft_helper.lora_alpha == 32
target_modules = sorted(peft_helper.target_modules)
assert target_modules == [
"down_proj",
"embed_tokens",
"gate_proj",
"k_proj",
"lm_head",
"o_proj",
"q_proj",
"up_proj",
"v_proj",
]
assert peft_helper.vllm_max_position_embeddings == 4096
# test RSLoRA
rslora_config = dict(use_rslora=True)
test_dir = tmp_path / "test_rslora"
shutil.copytree(llama32_lora_files, test_dir)
# Load and modify configuration
config_path = test_dir / "adapter_config.json"
with open(config_path) as f:
adapter_config = json.load(f)
# Apply configuration changes
adapter_config.update(rslora_config)
# Save modified configuration
with open(config_path, "w") as f:
json.dump(adapter_config, f)
peft_helper = PEFTHelper.from_local_dir(test_dir, max_position_embeddings=4096)
peft_helper.validate_legal(lora_config)
scaling = peft_helper.lora_alpha / math.sqrt(peft_helper.r)
assert abs(peft_helper.vllm_lora_scaling_factor - scaling) < 1e-3
@pytest.mark.parametrize("test_name,config_change,expected_error", ERROR_CASES)
def test_peft_helper_error(
llama32_lora_files,
tmp_path,
test_name: str,
config_change: dict,
expected_error: str,
):
test_dir = tmp_path / test_name
shutil.copytree(llama32_lora_files, test_dir)
# Load and modify configuration
config_path = test_dir / "adapter_config.json"
with open(config_path) as f:
adapter_config = json.load(f)
# Apply configuration changes
adapter_config.update(config_change)
# Save modified configuration
with open(config_path, "w") as f:
json.dump(adapter_config, f)
lora_config = LoRAConfig(max_lora_rank=16, max_cpu_loras=3, max_loras=2)
# Test loading the adapter
with pytest.raises(ValueError, match=expected_error):
PEFTHelper.from_local_dir(
test_dir, max_position_embeddings=4096
).validate_legal(lora_config)
@pytest.mark.parametrize("bad_rank", [0, -1, -8])
def test_peft_helper_invalid_rank_direct(bad_rank: int):
"""Regression test: constructing a PEFTHelper with a non-positive rank
must raise a clear ValueError instead of crashing with an unrelated
ZeroDivisionError (r=0) or silently succeeding with a sign-flipped
scaling factor that validate_legal() never catches (r<0, since its only
rank check is the upper bound against max_lora_rank).
Network-free: constructs PEFTHelper directly rather than going through
from_local_dir(), which needs an on-disk adapter_config.json.
"""
with pytest.raises(ValueError, match="must be a positive integer"):
PEFTHelper(r=bad_rank, lora_alpha=16, target_modules=["q_proj"])
@pytest.mark.parametrize(
"init_lora_weights",
[True, False, "gaussian", "eva", "orthogonal", "mica", "lora_ga"],
)
def test_peft_helper_init_lora_weights_supported(init_lora_weights):
"""A saved adapter with these inits loads in PEFT as a plain LoRA."""
lora_config = LoRAConfig(max_lora_rank=16, max_cpu_loras=3, max_loras=2)
PEFTHelper.from_dict(
{
"r": 8,
"lora_alpha": 16,
"target_modules": ["q_proj"],
"init_lora_weights": init_lora_weights,
"alora_invocation_tokens": None,
"layer_replication": None,
"use_bdlora": None,
}
).validate_legal(lora_config)
@pytest.mark.parametrize(
"config,module_name,expected",
[
# rank_pattern only: alpha / r_m
({"rank_pattern": {"q_proj": 4}}, "model.layers.0.self_attn.q_proj", (4, 8.0)),
({"rank_pattern": {"q_proj": 4}}, "model.layers.0.self_attn.v_proj", (16, 2.0)),
# alpha_pattern only: alpha_m / r
(
{"alpha_pattern": {"v_proj": 64}},
"model.layers.0.self_attn.v_proj",
(16, 4.0),
),
# keys only match at a "." boundary, like in PEFT
({"rank_pattern": {"proj": 4}}, "model.layers.0.self_attn.q_proj", (16, 2.0)),
# fused MoE experts from PEFT `target_parameters`: gate_up_proj is stored
# as `experts.base_layer`, down_proj as `experts`
(
{"rank_pattern": {"experts.gate_up_proj": 2}},
"model.layers.0.mlp.experts.base_layer",
(2, 16.0),
),
(
{"rank_pattern": {"experts.down_proj": 2}},
"model.layers.0.mlp.experts",
(2, 16.0),
),
(
{"rank_pattern": {"experts.gate_up_proj": 2}},
"model.layers.0.mlp.experts",
(16, 2.0),
),
],
)
def test_peft_helper_rank_and_alpha_pattern(config, module_name, expected):
peft_helper = PEFTHelper(
r=16, lora_alpha=32, target_modules=["q_proj", "v_proj"], **config
)
rank, scaling = peft_helper.get_rank_and_scaling(module_name)
assert rank == expected[0]
assert scaling == pytest.approx(expected[1])
def test_peft_helper_pattern_rslora():
peft_helper = PEFTHelper(
r=16,
lora_alpha=32,
target_modules=["q_proj"],
use_rslora=True,
rank_pattern={"q_proj": 4},
)
assert peft_helper.get_rank_and_scaling("model.q_proj") == (4, 32 / math.sqrt(4))
def test_peft_helper_dynamic_rank_conversion():
"""PEFT's `save_as_lora` with a dynamic rank writes r=1, lora_alpha=1 and the
same per-module values in rank_pattern and alpha_pattern, so every module
keeps a scaling of 1."""
peft_helper = PEFTHelper(
r=1,
lora_alpha=1,
target_modules=["q_proj", "v_proj"],
rank_pattern={"layers.0.self_attn.q_proj": 32, "layers.0.self_attn.v_proj": 3},
alpha_pattern={"layers.0.self_attn.q_proj": 32, "layers.0.self_attn.v_proj": 3},
)
assert peft_helper.get_rank_and_scaling("model.layers.0.self_attn.q_proj") == (
32,
1.0,
)
assert peft_helper.get_rank_and_scaling("model.layers.0.self_attn.v_proj") == (
3,
1.0,
)
with pytest.raises(ValueError, match="LoRA rank 32 is greater than max_lora_rank"):
peft_helper.validate_legal(LoRAConfig(max_lora_rank=16))
def test_peft_helper_null_patterns():
peft_helper = PEFTHelper.from_dict(
{
"r": 8,
"lora_alpha": 16,
"target_modules": ["q_proj"],
"rank_pattern": None,
"alpha_pattern": None,
}
)
assert peft_helper.get_rank_and_scaling("model.q_proj") == (8, 2.0)
@pytest.mark.parametrize("bad_rank", [0, -4, 2.5])
def test_peft_helper_invalid_rank_pattern(bad_rank):
with pytest.raises(ValueError, match="rank_pattern"):
PEFTHelper(
r=8,
lora_alpha=16,
target_modules=["q_proj"],
rank_pattern={"q_proj": bad_rank},
)
tests/lora/test_resolver.py
-75
View File
@@ -1,75 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm.lora.request import LoRARequest
from vllm.lora.resolver import LoRAResolver, LoRAResolverRegistry
class DummyLoRAResolver(LoRAResolver):
"""A dummy LoRA resolver for testing."""
async def resolve_lora(
self, base_model_name: str, lora_name: str
) -> LoRARequest | None:
if lora_name == "test_lora":
return LoRARequest(
lora_name=lora_name,
lora_path=f"/dummy/path/{base_model_name}/{lora_name}",
lora_int_id=abs(hash(lora_name)),
)
return None
def test_resolver_registry_registration():
"""Test basic resolver registration functionality."""
registry = LoRAResolverRegistry
resolver = DummyLoRAResolver()
# Register a new resolver
registry.register_resolver("dummy", resolver)
assert "dummy" in registry.get_supported_resolvers()
# Get registered resolver
retrieved_resolver = registry.get_resolver("dummy")
assert retrieved_resolver is resolver
def test_resolver_registry_duplicate_registration():
"""Test registering a resolver with an existing name."""
registry = LoRAResolverRegistry
resolver1 = DummyLoRAResolver()
resolver2 = DummyLoRAResolver()
registry.register_resolver("dummy", resolver1)
registry.register_resolver("dummy", resolver2)
assert registry.get_resolver("dummy") is resolver2
def test_resolver_registry_unknown_resolver():
"""Test getting a non-existent resolver."""
registry = LoRAResolverRegistry
with pytest.raises(KeyError, match="not found"):
registry.get_resolver("unknown_resolver")
@pytest.mark.asyncio
async def test_dummy_resolver_resolve():
"""Test the dummy resolver's resolve functionality."""
dummy_resolver = DummyLoRAResolver()
base_model_name = "base_model_test"
lora_name = "test_lora"
# Test successful resolution
result = await dummy_resolver.resolve_lora(base_model_name, lora_name)
assert isinstance(result, LoRARequest)
assert result.lora_name == lora_name
assert result.lora_path == f"/dummy/path/{base_model_name}/{lora_name}"
# Test failed resolution
result = await dummy_resolver.resolve_lora(base_model_name, "nonexistent_lora")
assert result is None
tests/lora/test_utils.py
-203
View File
@@ -1,203 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections import OrderedDict
from typing import NamedTuple
from unittest.mock import MagicMock, patch
import pytest
from huggingface_hub.utils import HfHubHTTPError
from torch import nn
from vllm.lora.utils import (
get_adapter_absolute_path,
parse_fine_tuned_lora_name,
replace_submodule,
)
from vllm.model_executor.models.utils import WeightsMapper
class LoRANameParserTestConfig(NamedTuple):
name: str
module_name: str
is_lora_a: bool
weights_mapper: WeightsMapper | None = None
def test_parse_fine_tuned_lora_name_valid():
fixture = [
LoRANameParserTestConfig(
"base_model.model.lm_head.lora_A.weight", "lm_head", True, False
),
LoRANameParserTestConfig(
"base_model.model.lm_head.lora_B.weight", "lm_head", False, False
),
LoRANameParserTestConfig(
"base_model.model.model.embed_tokens.lora_embedding_A",
"model.embed_tokens",
True,
),
LoRANameParserTestConfig(
"base_model.model.model.embed_tokens.lora_embedding_B",
"model.embed_tokens",
False,
),
LoRANameParserTestConfig(
"base_model.model.model.layers.9.mlp.down_proj.lora_A.weight",
"model.layers.9.mlp.down_proj",
True,
),
LoRANameParserTestConfig(
"base_model.model.model.layers.9.mlp.down_proj.lora_B.weight",
"model.layers.9.mlp.down_proj",
False,
),
LoRANameParserTestConfig(
"language_model.layers.9.mlp.down_proj.lora_A.weight",
"language_model.layers.9.mlp.down_proj",
True,
),
LoRANameParserTestConfig(
"language_model.layers.9.mlp.down_proj.lora_B.weight",
"language_model.layers.9.mlp.down_proj",
False,
),
# Test with WeightsMapper
LoRANameParserTestConfig(
"base_model.model.model.layers.9.mlp.down_proj.lora_A.weight",
"language_model.model.layers.9.mlp.down_proj",
True,
weights_mapper=WeightsMapper(
orig_to_new_prefix={"model.": "language_model.model."}
),
),
LoRANameParserTestConfig(
"base_model.model.model.layers.9.mlp.down_proj.lora_B.weight",
"language_model.model.layers.9.mlp.down_proj",
False,
weights_mapper=WeightsMapper(
orig_to_new_prefix={"model.": "language_model.model."}
),
),
LoRANameParserTestConfig(
"model.layers.9.mlp.down_proj.lora_A.weight",
"language_model.model.layers.9.mlp.down_proj",
True,
weights_mapper=WeightsMapper(
orig_to_new_prefix={"model.": "language_model.model."}
),
),
LoRANameParserTestConfig(
"model.layers.9.mlp.down_proj.lora_B.weight",
"language_model.model.layers.9.mlp.down_proj",
False,
weights_mapper=WeightsMapper(
orig_to_new_prefix={"model.": "language_model.model."}
),
),
]
for name, module_name, is_lora_a, weights_mapper in fixture:
assert (module_name, is_lora_a) == parse_fine_tuned_lora_name(
name, weights_mapper
)
def test_parse_fine_tuned_lora_name_invalid():
fixture = {
"base_model.weight",
"base_model.model.weight",
"base_model.model.model.layers.9.mlp.down_proj.lora_A.bias",
"base_model.model.model.layers.9.mlp.down_proj.lora_B.bias",
}
for name in fixture:
with pytest.raises(ValueError, match="unsupported LoRA weight"):
parse_fine_tuned_lora_name(name)
def test_replace_submodule():
model = nn.Sequential(
OrderedDict(
[
("dense1", nn.Linear(764, 100)),
("act1", nn.ReLU()),
("dense2", nn.Linear(100, 50)),
(
"seq1",
nn.Sequential(
OrderedDict(
[
("dense1", nn.Linear(100, 10)),
("dense2", nn.Linear(10, 50)),
]
)
),
),
("act2", nn.ReLU()),
("output", nn.Linear(50, 10)),
("outact", nn.Sigmoid()),
]
)
)
sigmoid = nn.Sigmoid()
replace_submodule(model, "act1", sigmoid)
assert dict(model.named_modules())["act1"] == sigmoid
dense2 = nn.Linear(1, 5)
replace_submodule(model, "seq1.dense2", dense2)
assert dict(model.named_modules())["seq1.dense2"] == dense2
# Unit tests for get_adapter_absolute_path
@patch("os.path.isabs")
def test_get_adapter_absolute_path_absolute(mock_isabs):
path = "/absolute/path/to/lora"
mock_isabs.return_value = True
assert get_adapter_absolute_path(path) == path
@patch("os.path.expanduser")
def test_get_adapter_absolute_path_expanduser(mock_expanduser):
# Path with ~ that needs to be expanded
path = "~/relative/path/to/lora"
absolute_path = "/home/user/relative/path/to/lora"
mock_expanduser.return_value = absolute_path
assert get_adapter_absolute_path(path) == absolute_path
@patch("os.path.exists")
@patch("os.path.abspath")
def test_get_adapter_absolute_path_local_existing(mock_abspath, mock_exist):
# Relative path that exists locally
path = "relative/path/to/lora"
absolute_path = "/absolute/path/to/lora"
mock_exist.return_value = True
mock_abspath.return_value = absolute_path
assert get_adapter_absolute_path(path) == absolute_path
@patch("huggingface_hub.HfApi.snapshot_download")
@patch("os.path.exists")
def test_get_adapter_absolute_path_huggingface(mock_exist, mock_snapshot_download):
# Hugging Face model identifier
path = "org/repo"
absolute_path = "/mock/snapshot/path"
mock_exist.return_value = False
mock_snapshot_download.return_value = absolute_path
assert get_adapter_absolute_path(path) == absolute_path
@patch("huggingface_hub.HfApi.snapshot_download")
@patch("os.path.exists")
def test_get_adapter_absolute_path_huggingface_error(
mock_exist, mock_snapshot_download
):
# Hugging Face model identifier with download error
path = "org/repo"
mock_exist.return_value = False
mock_snapshot_download.side_effect = HfHubHTTPError(
"failed to query model info",
response=MagicMock(),
)
assert get_adapter_absolute_path(path) == path
tests/model_executor/test_qwen3_5_quantization.py
-1
View File
@@ -19,7 +19,6 @@ def test_qwen3_5_lm_head_receives_quant_config():
mock_vllm_config.cache_config.mamba_cache_mode = "align"
mock_vllm_config.scheduler_config = Mock()
mock_vllm_config.quant_config = mock_quant_config
mock_vllm_config.lora_config = None
mock_pp_group = Mock()
mock_pp_group.is_last_rank = True
tests/models/fixtures/ministral_3b_chat.json
-1
View File
File diff suppressed because one or more lines are too long
tests/models/fixtures/pixtral_chat.json
-1
View File
File diff suppressed because one or more lines are too long
tests/models/multimodal/generation/test_transformers_audio.py
-9
View File
@@ -8,7 +8,6 @@ from transformers import AutoModelForSeq2SeqLM
from vllm.assets.audio import AudioAsset
from vllm.envs import disable_envs_cache
from vllm.lora.request import LoRARequest
from vllm.multimodal.audio import AudioResampler
from ....conftest import HfRunner, VllmRunner
@@ -26,7 +25,6 @@ AUDIO_MODEL_SETTINGS: dict[str, dict[str, Any]] = {
"<|end_of_text|>\n"
"<|start_of_role|>assistant<|end_of_role|>"
),
"audio_lora_path": "ibm-granite/granite-speech-3.3-2b",
},
"nvidia/audio-flamingo-3-hf": {
"prompt": (
@@ -90,7 +88,6 @@ def test_transformers_audio_generation(
model_id: str,
):
settings = AUDIO_MODEL_SETTINGS[model_id]
audio_lora_path = settings.get("audio_lora_path")
audio, orig_sr = AUDIO_ASSET.audio_and_sample_rate
target_sr = settings.get("sampling_rate", orig_sr)
@@ -105,19 +102,13 @@ def test_transformers_audio_generation(
max_model_len=2048,
enforce_eager=True,
limit_mm_per_prompt={"audio": 1},
enable_lora=audio_lora_path is not None,
max_lora_rank=64,
**settings.get("vllm_runner_kwargs", {}),
) as vllm_model:
lora_request = (
LoRARequest("audio", 1, audio_lora_path) if audio_lora_path else None
)
vllm_outputs = vllm_model.generate_greedy_logprobs(
[settings["prompt"]],
128,
num_logprobs=10,
audios=[audio],
lora_request=lora_request,
)
with hf_runner(
tests/plugins/vllm_add_dummy_endpoint_plugin/setup.py
-15
View File
@@ -1,15 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from setuptools import setup
setup(
name="vllm_add_dummy_endpoint_plugin",
version="0.1",
packages=["vllm_add_dummy_endpoint_plugin"],
entry_points={
"vllm.endpoint_plugins": [
"dummy_admin_endpoint_plugin = vllm_add_dummy_endpoint_plugin:DummyAdminEndpointPlugin" # noqa
]
},
)
tests/plugins/vllm_add_dummy_endpoint_plugin/vllm_add_dummy_endpoint_plugin/__init__.py
-26
View File
@@ -1,26 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Worked example `vllm.endpoint_plugins` entry point.
Reports scheduler config via `collective_rpc`. Demonstrates the full
contract: `attach_router` registers the route at Phase A (`build_app`) and
`init_state` stashes the `EngineClient` the route handler needs at Phase B
(`init_app_state`).
"""
from fastapi import FastAPI, Request
class DummyAdminEndpointPlugin:
name = "dummy_admin_endpoint_plugin"
required_tasks: tuple[str, ...] | None = None
def attach_router(self, app: FastAPI) -> None:
@app.get("/v1/admin/scheduler_config")
async def scheduler_config(raw_request: Request):
engine_client = raw_request.app.state.dummy_engine_client
results = await engine_client.collective_rpc("get_scheduler_config")
return {"scheduler_config": results}
async def init_state(self, engine_client, state, args) -> None:
state.dummy_engine_client = engine_client
tests/plugins_tests/__init__.py
View File
tests/plugins_tests/gguf/__init__.py
View File
tests/plugins_tests/gguf/test_gguf_plugin_generate.py
-130
View File
@@ -1,130 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""E2E tests for GGUF plugin functionality."""
import os
from typing import NamedTuple
import pytest
from transformers import AutoTokenizer
from ...conftest import VllmRunner
from ...models.utils import check_logprobs_close
from ...utils import multi_gpu_test
os.environ["TOKENIZERS_PARALLELISM"] = "true"
MAX_MODEL_LEN = 1024
class GGUFTestConfig(NamedTuple):
original_model: str
gguf_model_path: str # Full path to .gguf file
QWEN3_CONFIG = GGUFTestConfig(
original_model="Qwen/Qwen3-0.6B",
gguf_model_path="unsloth/Qwen3-0.6B-GGUF:Q8_0",
)
OLMOE_CONFIG = GGUFTestConfig(
original_model="allenai/OLMoE-1B-7B-0125",
gguf_model_path="allenai/OLMoE-1B-7B-0125-GGUF:Q6_K",
)
MODELS = [
QWEN3_CONFIG,
OLMOE_CONFIG,
]
def check_model_outputs(
vllm_runner: type[VllmRunner],
prompts: list[str],
model: GGUFTestConfig,
dtype: str,
max_tokens: int,
num_logprobs: int,
tp_size: int,
):
tokenizer = AutoTokenizer.from_pretrained(model.original_model)
if tokenizer.chat_template is not None:
messages = [[{"role": "user", "content": prompt}] for prompt in prompts]
prompts = tokenizer.apply_chat_template(
messages, tokenize=False, add_generation_prompt=True
)
# Run gguf model.
with vllm_runner(
model_name=model.gguf_model_path,
enforce_eager=True,
tokenizer_name=model.original_model,
dtype=dtype,
max_model_len=MAX_MODEL_LEN,
tensor_parallel_size=tp_size,
) as gguf_model:
gguf_outputs = gguf_model.generate_greedy_logprobs(
prompts[:-1], max_tokens, num_logprobs
)
# Run unquantized model.
# Should run with tp=1, otherwise the test will stuck at
# nccl initialization.
with vllm_runner(
model_name=model.original_model,
enforce_eager=True, # faster tests
dtype=dtype,
max_model_len=MAX_MODEL_LEN,
tensor_parallel_size=1,
) as original_model:
original_outputs = original_model.generate_greedy_logprobs(
prompts[:-1], max_tokens, num_logprobs
)
check_logprobs_close(
outputs_0_lst=original_outputs,
outputs_1_lst=gguf_outputs,
name_0="original",
name_1="gguf",
)
@pytest.mark.parametrize("model", MODELS)
@pytest.mark.parametrize("dtype", ["bfloat16"])
@pytest.mark.parametrize("max_tokens", [32])
@pytest.mark.parametrize("num_logprobs", [8])
@pytest.mark.parametrize("tp_size", [1])
def test_models(
vllm_runner: type[VllmRunner],
example_prompts: list[str],
model: GGUFTestConfig,
dtype: str,
max_tokens: int,
num_logprobs: int,
tp_size: int,
) -> None:
check_model_outputs(
vllm_runner, example_prompts, model, dtype, max_tokens, num_logprobs, tp_size
)
@pytest.mark.parametrize("model", MODELS)
@pytest.mark.parametrize("dtype", ["half"])
@pytest.mark.parametrize("max_tokens", [8])
@pytest.mark.parametrize("num_logprobs", [5])
@pytest.mark.parametrize("tp_size", [2])
@multi_gpu_test(num_gpus=2)
def test_distributed(
vllm_runner: type[VllmRunner],
example_prompts: list[str],
model: GGUFTestConfig,
dtype: str,
max_tokens: int,
num_logprobs: int,
tp_size: int,
) -> None:
check_model_outputs(
vllm_runner, example_prompts, model, dtype, max_tokens, num_logprobs, tp_size
)
tests/plugins_tests/gguf/test_gguf_plugin_multimodal.py
-167
View File
@@ -1,167 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
os.environ["TOKENIZERS_PARALLELISM"] = "true"
from typing import Any, NamedTuple
import pytest
from pytest import MarkDecorator
from transformers import AutoModelForImageTextToText
from vllm.assets.image import ImageAsset
from vllm.multimodal.image import rescale_image_size
from vllm.transformers_utils.repo_utils import hf_api
from vllm.utils.torch_utils import set_default_torch_num_threads
from ...conftest import IMAGE_ASSETS, HfRunner, VllmRunner
from ...models.utils import check_logprobs_close
class GGUFMMTestConfig(NamedTuple):
original_model: str
gguf_repo: str
gguf_backbone: str
gguf_mmproj: str
prompt: list[str]
image_names: list[str] # Store names, load PIL images at runtime
max_model_len: int = 4096
marks: list[MarkDecorator] = []
mm_processor_kwargs: dict[str, Any] = {}
@property
def gguf_model(self):
hf_api().hf_hub_download(self.gguf_repo, filename=self.gguf_mmproj)
return hf_api().hf_hub_download(self.gguf_repo, filename=self.gguf_backbone)
# Common prompts aligned with test_common.py "gemma3" entry format
_GEMMA3_PROMPTS = IMAGE_ASSETS.prompts(
{
"stop_sign": (
"<bos><start_of_turn>user\n"
"<start_of_image>What's the content in the center of the image?"
"<end_of_turn>\n<start_of_turn>model\n"
),
"cherry_blossom": (
"<bos><start_of_turn>user\n"
"<start_of_image>What is the season?"
"<end_of_turn>\n<start_of_turn>model\n"
),
}
)
# Image asset names - load at runtime to avoid pickle issues with subprocess
_GEMMA3_IMAGE_NAMES = ["stop_sign", "cherry_blossom"]
# Regular multimodal (no pan-and-scan) - uses QAT Q4_0 GGUF
GEMMA3_CONFIG = GGUFMMTestConfig(
original_model="google/gemma-3-4b-it",
gguf_repo="google/gemma-3-4b-it-qat-q4_0-gguf",
gguf_backbone="gemma-3-4b-it-q4_0.gguf",
gguf_mmproj="mmproj-model-f16-4B.gguf",
prompt=_GEMMA3_PROMPTS,
image_names=_GEMMA3_IMAGE_NAMES,
max_model_len=4096,
mm_processor_kwargs={},
)
# Pan-and-scan multimodal - uses unquantized BF16 GGUF
GEMMA3_CONFIG_PAN_AND_SCAN = GGUFMMTestConfig(
original_model="google/gemma-3-4b-it",
gguf_repo="google/gemma-3-4b-it-qat-q4_0-gguf",
gguf_backbone="gemma-3-4b-it-q4_0.gguf",
gguf_mmproj="mmproj-model-f16-4B.gguf",
prompt=_GEMMA3_PROMPTS,
image_names=_GEMMA3_IMAGE_NAMES,
max_model_len=4096,
mm_processor_kwargs={"do_pan_and_scan": True},
)
MODELS_TO_TEST = [GEMMA3_CONFIG, GEMMA3_CONFIG_PAN_AND_SCAN]
def run_multimodal_gguf_test(
hf_runner: type[HfRunner],
vllm_runner: type[VllmRunner],
model: GGUFMMTestConfig,
dtype: str,
max_tokens: int,
num_logprobs: int,
):
# Load images at runtime (inside subprocess) to avoid pickle issues
images = [ImageAsset(name).pil_image for name in model.image_names]
size_factors = [0.25, 0.5, 1.0]
inputs_per_image = [
(
[prompt for _ in size_factors],
[rescale_image_size(image, factor) for factor in size_factors],
)
for image, prompt in zip(images, model.prompt)
]
# NOTE: Run vLLM first to avoid CUDA init issues with multiprocessing fork.
# Run GGUF model via vLLM.
with (
set_default_torch_num_threads(1),
vllm_runner(
model_name=model.gguf_model,
enforce_eager=True,
tokenizer_name=model.original_model,
dtype=dtype,
max_model_len=model.max_model_len,
mm_processor_kwargs=model.mm_processor_kwargs,
) as gguf_model,
):
gguf_outputs_per_case = [
gguf_model.generate_greedy_logprobs(
prompts,
max_tokens,
num_logprobs=num_logprobs,
images=images,
)
for prompts, images in inputs_per_image
]
# Then run HfRunner for HuggingFace baseline comparison.
with hf_runner(
model.original_model,
dtype=dtype,
auto_cls=AutoModelForImageTextToText,
) as hf_model:
hf_outputs_per_case = [
hf_model.generate_greedy_logprobs_limit(
prompts,
max_tokens,
num_logprobs=num_logprobs,
images=images,
)
for prompts, images in inputs_per_image
]
for hf_outputs, gguf_outputs in zip(hf_outputs_per_case, gguf_outputs_per_case):
check_logprobs_close(
outputs_0_lst=hf_outputs,
outputs_1_lst=gguf_outputs,
name_0="hf",
name_1="gguf",
)
@pytest.mark.parametrize("model", MODELS_TO_TEST)
@pytest.mark.parametrize("dtype", ["bfloat16"])
@pytest.mark.parametrize("max_tokens", [32])
@pytest.mark.parametrize("num_logprobs", [10])
def test_gemma3_mm_gguf(
hf_runner: type[HfRunner],
vllm_runner: type[VllmRunner],
model: GGUFMMTestConfig,
dtype: str,
max_tokens: int,
num_logprobs: int,
) -> None:
run_multimodal_gguf_test(
hf_runner, vllm_runner, model, dtype, max_tokens, num_logprobs
)
tests/plugins_tests/lora_resolvers/__init__.py
View File
tests/plugins_tests/lora_resolvers/test_filesystem_resolver.py
-65
View File
@@ -1,65 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
import shutil
import pytest
from vllm.plugins.lora_resolvers.filesystem_resolver import FilesystemResolver
from vllm.transformers_utils.repo_utils import hf_api
MODEL_NAME = "Qwen/Qwen3-0.6B"
LORA_NAME = "charent/self_cognition_Alice"
PA_NAME = "swapnilbp/llama_tweet_ptune"
@pytest.fixture(scope="module")
def adapter_cache(request, tmpdir_factory):
# Create dir that mimics the structure of the adapter cache
adapter_cache = tmpdir_factory.mktemp(request.module.__name__) / "adapter_cache"
return adapter_cache
@pytest.fixture(scope="module")
def qwen3_lora_files():
return hf_api().snapshot_download(repo_id=LORA_NAME)
@pytest.fixture(scope="module")
def pa_files():
return hf_api().snapshot_download(repo_id=PA_NAME)
@pytest.mark.asyncio
async def test_filesystem_resolver(adapter_cache, qwen3_lora_files):
model_files = adapter_cache / LORA_NAME
shutil.copytree(qwen3_lora_files, model_files)
fs_resolver = FilesystemResolver(adapter_cache)
assert fs_resolver is not None
lora_request = await fs_resolver.resolve_lora(MODEL_NAME, LORA_NAME)
assert lora_request is not None
assert lora_request.lora_name == LORA_NAME
assert lora_request.lora_path == os.path.join(adapter_cache, LORA_NAME)
@pytest.mark.asyncio
async def test_missing_adapter(adapter_cache):
fs_resolver = FilesystemResolver(adapter_cache)
assert fs_resolver is not None
missing_lora_request = await fs_resolver.resolve_lora(MODEL_NAME, "foobar")
assert missing_lora_request is None
@pytest.mark.asyncio
async def test_nonlora_adapter(adapter_cache, pa_files):
model_files = adapter_cache / PA_NAME
shutil.copytree(pa_files, model_files)
fs_resolver = FilesystemResolver(adapter_cache)
assert fs_resolver is not None
pa_request = await fs_resolver.resolve_lora(MODEL_NAME, PA_NAME)
assert pa_request is None
tests/plugins_tests/lora_resolvers/test_hf_hub_resolver.py
-108
View File
@@ -1,108 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
import pytest
from huggingface_hub.constants import HF_HUB_CACHE
from vllm.plugins.lora_resolvers.hf_hub_resolver import HfHubResolver
LORA_LIB_MODEL_NAME = "ibm-granite/granite-3.3-8b-instruct"
# Repo with multiple LoRAs contained in it
LORA_LIB = "ibm-granite/granite-3.3-8b-rag-agent-lib"
LORA_NAME = "ibm-granite/granite-3.3-8b-rag-agent-lib/answerability_prediction_lora" # noqa: E501
NON_LORA_SUBPATH = "ibm-granite/granite-3.3-8b-rag-agent-lib/README.md"
LIB_DOWNLOAD_DIR = os.path.join(
HF_HUB_CACHE, "models--ibm-granite--granite-3.3-8b-rag-agent-lib"
)
INVALID_REPO_NAME = "thisrepodoesnotexist"
# Repo with only one LoRA in the root dir
LORA_REPO_MODEL_NAME = "meta-llama/Llama-2-7b-hf"
LORA_REPO = "yard1/llama-2-7b-sql-lora-test"
REPO_DOWNLOAD_DIR = os.path.join(
HF_HUB_CACHE, "models--yard1--llama-2-7b-sql-lora-test"
)
@pytest.mark.asyncio
async def test_hf_resolver_with_direct_path():
hf_resolver = HfHubResolver([LORA_REPO])
assert hf_resolver is not None
lora_request = await hf_resolver.resolve_lora(LORA_REPO_MODEL_NAME, LORA_REPO)
assert lora_request is not None
assert lora_request.lora_name == LORA_REPO
assert REPO_DOWNLOAD_DIR in lora_request.lora_path
assert "adapter_config.json" in os.listdir(lora_request.lora_path)
@pytest.mark.asyncio
async def test_hf_resolver_with_nested_paths():
hf_resolver = HfHubResolver([LORA_LIB])
assert hf_resolver is not None
lora_request = await hf_resolver.resolve_lora(LORA_LIB_MODEL_NAME, LORA_NAME)
assert lora_request is not None
assert lora_request.lora_name == LORA_NAME
assert LIB_DOWNLOAD_DIR in lora_request.lora_path
assert "adapter_config.json" in os.listdir(lora_request.lora_path)
@pytest.mark.asyncio
async def test_hf_resolver_with_multiple_repos():
hf_resolver = HfHubResolver([LORA_LIB, LORA_REPO])
assert hf_resolver is not None
lora_request = await hf_resolver.resolve_lora(LORA_LIB_MODEL_NAME, LORA_NAME)
assert lora_request is not None
assert lora_request.lora_name == LORA_NAME
assert LIB_DOWNLOAD_DIR in lora_request.lora_path
assert "adapter_config.json" in os.listdir(lora_request.lora_path)
@pytest.mark.asyncio
async def test_missing_adapter():
hf_resolver = HfHubResolver([LORA_LIB])
assert hf_resolver is not None
missing_lora_request = await hf_resolver.resolve_lora(LORA_LIB_MODEL_NAME, "foobar")
assert missing_lora_request is None
@pytest.mark.asyncio
async def test_nonlora_adapter():
hf_resolver = HfHubResolver([LORA_LIB])
assert hf_resolver is not None
readme_request = await hf_resolver.resolve_lora(
LORA_LIB_MODEL_NAME, NON_LORA_SUBPATH
)
assert readme_request is None
@pytest.mark.asyncio
async def test_invalid_repo():
hf_resolver = HfHubResolver([LORA_LIB])
assert hf_resolver is not None
invalid_repo_req = await hf_resolver.resolve_lora(
INVALID_REPO_NAME,
f"{INVALID_REPO_NAME}/foo",
)
assert invalid_repo_req is None
@pytest.mark.asyncio
async def test_trailing_slash():
hf_resolver = HfHubResolver([LORA_LIB])
assert hf_resolver is not None
lora_request = await hf_resolver.resolve_lora(
LORA_LIB_MODEL_NAME,
f"{LORA_NAME}/",
)
assert lora_request is not None
assert lora_request.lora_name == f"{LORA_NAME}/"
assert LIB_DOWNLOAD_DIR in lora_request.lora_path
assert "adapter_config.json" in os.listdir(lora_request.lora_path)
tests/plugins_tests/test_endpoint_plugins.py
-363
View File
@@ -1,363 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the `vllm.endpoint_plugins` framework (RFC #46565).
Uses the worked in repo example plugin (`vllm_add_dummy_endpoint_plugin`,
installed via `tests/plugins/vllm_add_dummy_endpoint_plugin`) exercising both
`EndpointPlugin` hooks against a fake `EngineClient`, unit tests for the
`load_endpoint_plugins` gating matrix and an e2e test that drives a real HTTP
request through the plugin's route.
"""
from argparse import Namespace
from typing import Any
import httpx
import pytest
from fastapi import APIRouter, FastAPI
from vllm_add_dummy_endpoint_plugin import DummyAdminEndpointPlugin
from vllm.entrypoints.launchers.app import build_app
from vllm.entrypoints.launchers.cli_args import make_arg_parser
from vllm.plugins import load_endpoint_plugins
from vllm.plugins.endpoint_plugins.interface import (
EndpointPlugin,
attach_endpoint_plugins,
init_endpoint_plugins_state,
)
from vllm.utils.argparse_utils import FlexibleArgumentParser
class _RaisingEndpointPlugin:
"""Factory that raises to exercise the "instantiation fails" path."""
name = "raising_endpoint_plugin"
required_tasks = None
def __init__(self):
raise RuntimeError("boom")
class _ShadowEndpointPlugin:
name = "shadow_endpoint_plugin"
required_tasks = None
def __init__(self, path: str, include_in_schema: bool):
self.path = path
self.include_in_schema = include_in_schema
def attach_router(self, app: FastAPI) -> None:
assert str(app.url_path_for("show_version")) == "/version"
app.openapi()
@app.get(self.path, status_code=209, include_in_schema=self.include_in_schema)
async def plugin_version(plugin_arg: str) -> dict[str, str]:
return {"served_by": plugin_arg}
async def init_state(self, engine_client, state, args) -> None:
pass
class _RouterEndpointPlugin(DummyAdminEndpointPlugin):
def __init__(self, router: APIRouter):
self.router = router
def attach_router(self, app: FastAPI) -> None:
app.include_router(self.router)
class _FakeEngineClient:
"""Minimal stand in exercising `collective_rpc`. Not a real engine."""
def __init__(self, rpc_result: Any = None):
self.rpc_result = rpc_result
self.rpc_calls: list[tuple[str, tuple, dict]] = []
async def collective_rpc(self, method, timeout=None, args=(), kwargs=None):
self.rpc_calls.append((method, args, kwargs or {}))
return self.rpc_result
def _build_args() -> Namespace:
parser = FlexibleArgumentParser()
subparsers = parser.add_subparsers()
serve_parser = subparsers.add_parser("serve")
make_arg_parser(serve_parser)
return serve_parser.parse_args([])
def _fake_loader(factories: dict[str, Any]):
def _load_plugins_by_group(group: str) -> dict[str, Any]:
assert group == "vllm.endpoint_plugins"
return factories
return _load_plugins_by_group
def test_dummy_plugin_satisfies_protocol():
assert isinstance(DummyAdminEndpointPlugin(), EndpointPlugin)
def test_no_plugins_loaded_when_allowlist_unset(monkeypatch: pytest.MonkeyPatch):
monkeypatch.delenv("VLLM_PLUGINS", raising=False)
assert load_endpoint_plugins(("generate",)) == []
def test_no_plugins_loaded_when_allowlist_is_empty_string(
monkeypatch: pytest.MonkeyPatch,
):
"""`VLLM_PLUGINS=""` parses to `[""]`, not `None` (see `vllm.envs`), so it
must be treated as a (non strict) allowlist matching no plugin name, not
as "unset"."""
monkeypatch.setenv("VLLM_PLUGINS", "")
assert load_endpoint_plugins(("generate",)) == []
def test_plugin_loaded_when_allowlisted_and_task_matches(
monkeypatch: pytest.MonkeyPatch,
):
monkeypatch.setenv("VLLM_PLUGINS", "dummy_admin_endpoint_plugin")
plugins = load_endpoint_plugins(("generate",))
assert len(plugins) == 1
assert isinstance(plugins[0], DummyAdminEndpointPlugin)
def test_plugin_skipped_when_required_tasks_miss(monkeypatch: pytest.MonkeyPatch):
class _GenerateOnlyPlugin(DummyAdminEndpointPlugin):
required_tasks = ("generate",)
monkeypatch.setenv("VLLM_PLUGINS", "dummy_admin_endpoint_plugin")
monkeypatch.setattr(
"vllm.plugins.load_plugins_by_group",
_fake_loader({"dummy_admin_endpoint_plugin": _GenerateOnlyPlugin}),
)
assert load_endpoint_plugins(("embed",)) == []
assert len(load_endpoint_plugins(("generate",))) == 1
def test_plugin_loaded_when_required_tasks_is_none(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setenv("VLLM_PLUGINS", "dummy_admin_endpoint_plugin")
assert len(load_endpoint_plugins(supported_tasks=None)) == 1
def test_plugin_skipped_when_required_tasks_set_but_supported_tasks_none(
monkeypatch: pytest.MonkeyPatch,
):
class _GenerateOnlyPlugin(DummyAdminEndpointPlugin):
required_tasks = ("generate",)
monkeypatch.setenv("VLLM_PLUGINS", "dummy_admin_endpoint_plugin")
monkeypatch.setattr(
"vllm.plugins.load_plugins_by_group",
_fake_loader({"dummy_admin_endpoint_plugin": _GenerateOnlyPlugin}),
)
assert load_endpoint_plugins(supported_tasks=None) == []
def test_factory_raising_is_logged_and_skipped(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setenv(
"VLLM_PLUGINS", "raising_endpoint_plugin,dummy_admin_endpoint_plugin"
)
monkeypatch.setattr(
"vllm.plugins.load_plugins_by_group",
_fake_loader(
{
"raising_endpoint_plugin": _RaisingEndpointPlugin,
"dummy_admin_endpoint_plugin": DummyAdminEndpointPlugin,
}
),
)
plugins = load_endpoint_plugins(("generate",))
assert len(plugins) == 1
assert isinstance(plugins[0], DummyAdminEndpointPlugin)
def test_attach_is_noop_when_nothing_discovered(monkeypatch: pytest.MonkeyPatch):
monkeypatch.delenv("VLLM_PLUGINS", raising=False)
app = FastAPI()
attach_endpoint_plugins(app, ("generate",))
assert app.state.endpoint_plugins == []
@pytest.mark.asyncio
@pytest.mark.parametrize("path", ["/version", "/metrics"])
@pytest.mark.parametrize("include_in_schema", [True, False])
async def test_endpoint_plugin_override_preserves_hook_and_schema(
monkeypatch: pytest.MonkeyPatch, path: str, include_in_schema: bool
):
"""Hooks see core routes; overrides win over both HTTP routes and mounts."""
args = _build_args()
monkeypatch.setenv("VLLM_PLUGINS", "shadow_endpoint_plugin")
monkeypatch.setattr(
"vllm.plugins.load_plugins_by_group",
_fake_loader(
{
"shadow_endpoint_plugin": lambda: _ShadowEndpointPlugin(
path, include_in_schema
)
}
),
)
app = build_app(args, supported_tasks=())
transport = httpx.ASGITransport(app=app)
async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
response = await client.get(path, params={"plugin_arg": "plugin"})
schema = (await client.get("/openapi.json")).json()
assert response.status_code == 209
assert response.json() == {"served_by": "plugin"}
if include_in_schema:
operation = schema["paths"][path]["get"]
assert "209" in operation["responses"]
assert "200" not in operation["responses"]
assert operation["parameters"][0]["name"] == "plugin_arg"
assert operation["parameters"][0]["required"] is True
response_schema = operation["responses"]["209"]["content"]["application/json"][
"schema"
]
assert response_schema["additionalProperties"] == {"type": "string"}
else:
assert path not in schema["paths"]
@pytest.mark.asyncio
async def test_endpoint_plugin_override_preserves_other_methods(
monkeypatch: pytest.MonkeyPatch,
):
"""Overriding GET preserves POST's validation and the shared core route."""
core = APIRouter()
@core.api_route("/shared", methods=["GET", "POST"], status_code=201)
async def original(value: int) -> dict[str, int]:
return {"value": value}
plugin = APIRouter()
@plugin.get("/shared", status_code=202)
async def replacement():
return {"served_by": "plugin"}
app = FastAPI()
app.include_router(core)
original_route = app.routes[-1]
monkeypatch.setattr(
"vllm.plugins.load_endpoint_plugins",
lambda supported_tasks: [_RouterEndpointPlugin(plugin)],
)
attach_endpoint_plugins(app, ())
transport = httpx.ASGITransport(app=app)
async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
response = await client.get("/shared")
post_response = await client.post("/shared", params={"value": 7})
invalid_response = await client.post("/shared", params={"value": "invalid"})
assert response.status_code == 202
assert response.json() == {"served_by": "plugin"}
assert post_response.status_code == 201
assert post_response.json() == {"value": 7}
assert invalid_response.status_code == 422
assert original_route.methods == {"GET", "POST"}
assert core.routes[0].methods == {"GET", "POST"}
operations = app.openapi()["paths"]["/shared"]
assert "202" in operations["get"]["responses"]
assert "201" in operations["post"]["responses"]
@pytest.mark.asyncio
async def test_later_endpoint_plugin_overrides_same_operation(
monkeypatch: pytest.MonkeyPatch,
):
"""The last registered override wins while unrelated plugin routes survive."""
first = APIRouter()
@first.get("/shared", status_code=201)
@first.get("/plugins/first")
async def first_handler():
return {"served_by": "first"}
second = APIRouter()
@second.get("/shared", status_code=202)
async def second_handler():
return {"served_by": "second"}
monkeypatch.setattr(
"vllm.plugins.load_endpoint_plugins",
lambda supported_tasks: [
_RouterEndpointPlugin(first),
_RouterEndpointPlugin(second),
],
)
app = FastAPI()
attach_endpoint_plugins(app, ())
transport = httpx.ASGITransport(app=app)
async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
response = await client.get("/shared")
unrelated = await client.get("/plugins/first")
unsupported = await client.delete("/shared")
assert response.status_code == 202
assert response.json() == {"served_by": "second"}
assert unrelated.status_code == 200
assert unrelated.json() == {"served_by": "first"}
assert unsupported.status_code == 405
responses = app.openapi()["paths"]["/shared"]["get"]["responses"]
assert "202" in responses
assert "201" not in responses
@pytest.mark.asyncio
async def test_init_state_is_noop_without_phase_a(monkeypatch: pytest.MonkeyPatch):
"""A caller that builds a bare `State()` without `build_app` must not crash
just because `state.endpoint_plugins` was never set."""
from starlette.datastructures import State
monkeypatch.setenv("VLLM_PLUGINS", "dummy_admin_endpoint_plugin")
state = State()
await init_endpoint_plugins_state(_FakeEngineClient(), state, _build_args())
assert not hasattr(state, "dummy_engine_client")
@pytest.mark.asyncio
async def test_endpoint_plugin_end_to_end(monkeypatch: pytest.MonkeyPatch):
"""Phase A (attach) + Phase B (init) wired through `build_app` then
exercised with a real HTTP request against the worked example plugin."""
monkeypatch.setenv("VLLM_PLUGINS", "dummy_admin_endpoint_plugin")
args = _build_args()
app = build_app(args, supported_tasks=())
assert len(app.state.endpoint_plugins) == 1
assert any(
getattr(route, "path", None) == "/v1/admin/scheduler_config"
for route in app.routes
)
fake_engine_client = _FakeEngineClient(rpc_result=["cfg-a", "cfg-b"])
await init_endpoint_plugins_state(fake_engine_client, app.state, args)
assert app.state.dummy_engine_client is fake_engine_client
transport = httpx.ASGITransport(app=app)
async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
response = await client.get("/v1/admin/scheduler_config")
assert response.status_code == 200
assert response.json() == {"scheduler_config": ["cfg-a", "cfg-b"]}
assert fake_engine_client.rpc_calls == [("get_scheduler_config", (), {})]
tests/plugins_tests/test_oot_registration_online.py
-43
View File
@@ -1,43 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from tests.utils import VLLM_PATH, RemoteOpenAIServer
chatml_jinja_path = VLLM_PATH / "examples/template_chatml.jinja"
assert chatml_jinja_path.exists()
def run_and_test_dummy_opt_api_server(model, tp=1, extra_args=(), env_dict=None):
# the model is registered through the plugin
server_args = [
"--gpu-memory-utilization",
"0.10",
"--dtype",
"float32",
"--chat-template",
str(chatml_jinja_path),
"--load-format",
"dummy",
"-tp",
f"{tp}",
*extra_args,
]
with RemoteOpenAIServer(model, server_args, env_dict=env_dict) as server:
client = server.get_client()
completion = client.chat.completions.create(
model=model,
messages=[
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": "Hello!"},
],
temperature=0,
)
generated_text = completion.choices[0].message.content
assert generated_text is not None
# make sure only the first token is generated
rest = generated_text.replace("<s>", "")
assert rest == ""
def test_oot_registration_for_api_server(dummy_opt_path: str):
run_and_test_dummy_opt_api_server(dummy_opt_path)
tests/test_config.py
-1
View File
@@ -1049,7 +1049,6 @@ def test_resolve_cudagraph_mode_skips_mamba_block_check_while_profiling():
def test_adaptive_verification_requires_full_cudagraphs(graph_mode, should_raise):
config = SimpleNamespace(
speculative_config=SimpleNamespace(enable_adaptive_verification=True),
lora_config=None,
compilation_config=CompilationConfig(cudagraph_mode=graph_mode),
parallel_config=SimpleNamespace(pipeline_parallel_size=1),
)
tests/v1/core/test_output.py
-1
View File
@@ -22,7 +22,6 @@ def _create_new_requests_data(prompt_embeds: torch.Tensor | None) -> NewRequestD
pooling_params=None,
block_ids=([],),
num_computed_tokens=0,
lora_request=None,
prompt_embeds=prompt_embeds,
)
tests/v1/core/test_prefix_caching.py
-65
View File
@@ -19,7 +19,6 @@ from vllm.distributed.kv_events import (
BlockRemoved,
BlockStored,
)
from vllm.lora.request import LoRARequest
from vllm.multimodal.inputs import (
MultiModalFeatureSpec,
MultiModalKwargsItem,
@@ -74,7 +73,6 @@ def make_request(
mm_hashes: list[str] | None = None,
prompt_logprobs: int | None = None,
cache_salt: str | None = None,
lora_request: LoRARequest | None = None,
session_id: str | None = None,
):
mm_features = []
@@ -98,7 +96,6 @@ def make_request(
mm_features=mm_features if mm_features else None,
sampling_params=sampling_params,
pooling_params=None,
lora_request=lora_request,
cache_salt=cache_salt,
block_hasher=get_request_block_hasher(block_size, hash_fn),
session_id=session_id,
@@ -2356,64 +2353,6 @@ def test_null_parent_block_hash():
assert blocks[num_full_blocks - 1].block_hash is not None
@pytest.mark.parametrize("blocks_to_cache", [2, 3, 10])
def test_kv_cache_events_with_lora(blocks_to_cache: int):
"""Test BlockStored events contain correct lora_id when using LoRA requests."""
block_size = 16
num_blocks = blocks_to_cache + 1
# Create KVCacheManager with events enabled
manager = make_kv_cache_manager(
make_kv_cache_config(block_size, num_blocks),
max_model_len=8192,
enable_caching=True,
enable_kv_cache_events=True,
hash_block_size=block_size,
)
# Test with LoRA request
lora_request = LoRARequest(
lora_name="test_lora", lora_int_id=42, lora_path="/test/path"
)
num_tokens = block_size * blocks_to_cache
req_with_lora = make_request(
"lora_req",
list(range(num_tokens)),
block_size,
sha256,
lora_request=lora_request,
)
# Allocate slots and get events
_ = manager.allocate_slots(req_with_lora, num_tokens)
events = manager.take_events()
# Verify BlockStored event contains correct lora_id
block_stored_event = events[-1]
assert isinstance(block_stored_event, BlockStored)
assert block_stored_event.lora_id == 42 # Should match lora_request.adapter_id
assert len(block_stored_event.block_hashes) == blocks_to_cache
assert block_stored_event.block_size == block_size
# Clean up
manager.free(req_with_lora)
# Test without LoRA request (should have lora_id=None)
req_without_lora = make_request(
"no_lora_req", list(range(num_tokens)), block_size, sha256
)
_ = manager.allocate_slots(req_without_lora, num_tokens)
events = manager.take_events()
block_stored_event = events[-1]
assert isinstance(block_stored_event, BlockStored)
assert block_stored_event.lora_id is None # Should be None when no LoRA request
assert len(block_stored_event.block_hashes) == blocks_to_cache
assert block_stored_event.block_size == block_size
@pytest.mark.parametrize("group_id", [0, 1, 2])
def test_block_stored_event_group_idx(group_id: int):
"""Test BlockStored events emitted by cache_full_blocks carry the correct
@@ -2634,9 +2573,7 @@ def test_block_stored_event_group_idx_out_of_bounds(monkeypatch):
parent_block_hash=None,
token_ids=list(range(block_size)),
block_size=block_size,
lora_id=None,
medium=None,
lora_name=None,
group_idx=1,
)
manager.block_pool.kv_event_queue.append(event)
@@ -2764,8 +2701,6 @@ def test_emit_cached_block_events():
assert event.group_idx == kv_cache_group_id
assert event.block_size == block_size
assert event.medium == MEDIUM_GPU
assert event.lora_id is None
assert event.lora_name is None
assert event.session_id == "agent-session-reuse"
tests/v1/cudagraph/test_cudagraph_manager.py
+2 -12
View File
@@ -382,9 +382,7 @@ def _make_spec_decode_manager(
def test_uniform_decode_pads_up_to_full_graph(monkeypatch):
manager = _make_spec_decode_manager(monkeypatch)
assert [
(desc.cg_mode, desc.num_tokens) for desc in manager._candidates[(12, 0)]
] == [
assert [(desc.cg_mode, desc.num_tokens) for desc in manager._candidates[12]] == [
(CUDAGraphMode.FULL, 18),
(CUDAGraphMode.PIECEWISE, 16),
]
@@ -393,7 +391,6 @@ def test_uniform_decode_pads_up_to_full_graph(monkeypatch):
num_reqs=4,
num_tokens=12,
uniform_token_count=_DECODE_QUERY_LEN,
num_active_loras=0,
)
assert desc.cg_mode == CUDAGraphMode.FULL
@@ -409,7 +406,6 @@ def test_uniform_decode_exact_match_is_not_over_padded(monkeypatch):
num_reqs=3,
num_tokens=9,
uniform_token_count=_DECODE_QUERY_LEN,
num_active_loras=0,
)
assert desc.cg_mode == CUDAGraphMode.FULL
@@ -424,7 +420,6 @@ def test_mixed_batch_never_selects_a_uniform_decode_graph(monkeypatch):
num_reqs=2,
num_tokens=12,
uniform_token_count=None,
num_active_loras=0,
)
assert desc.cg_mode == CUDAGraphMode.PIECEWISE
@@ -443,7 +438,6 @@ def test_varlen_decode_graph_takes_bounded_decode_batches_only(monkeypatch):
num_reqs=3,
num_tokens=6,
uniform_token_count=None,
num_active_loras=0,
max_query_len=max_query_len,
)
@@ -479,7 +473,6 @@ def test_mixed_batch_at_decode_only_token_count_still_gets_a_graph(monkeypatch):
num_reqs=1,
num_tokens=num_tokens,
uniform_token_count=None,
num_active_loras=0,
)
assert desc.cg_mode == CUDAGraphMode.PIECEWISE, num_tokens
assert desc.uniform_token_count is None, num_tokens
@@ -493,7 +486,6 @@ def test_uniform_decode_beyond_capture_ladder_falls_back(monkeypatch):
num_reqs=9,
num_tokens=27,
uniform_token_count=_DECODE_QUERY_LEN,
num_active_loras=0,
)
assert desc.cg_mode == CUDAGraphMode.NONE
@@ -529,13 +521,12 @@ def test_dynamic_spec_decode_shared_token_count_stays_reachable(monkeypatch):
for desc in full_descs:
assert desc.num_reqs is not None
assert desc in manager._candidates[(desc.num_tokens, 0)], desc
assert desc in manager._candidates[desc.num_tokens], desc
assert (
manager.dispatch(
num_reqs=desc.num_reqs,
num_tokens=desc.num_tokens,
uniform_token_count=desc.uniform_token_count,
num_active_loras=0,
)
== desc
), desc
@@ -633,7 +624,6 @@ def test_divisor_query_len_dispatch_is_unchanged(
num_reqs=num_reqs,
num_tokens=num_tokens,
uniform_token_count=decode_query_len,
num_active_loras=0,
)
expected = min(s for s in capture_sizes if s >= num_tokens)
assert desc.cg_mode == CUDAGraphMode.FULL, num_tokens
tests/v1/determinism/test_lora_shrink_batch_invariant.py
-99
View File
@@ -1,99 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Batch-invariant LoRA shrink: fixed split-K partials, fixed-order reduction."""
import pytest
import torch
from utils import skip_if_not_cuda_alike
import vllm.lora.ops.triton_ops.lora_shrink_op as shrink_op
import vllm.lora.ops.triton_ops.utils as lora_utils
from vllm.lora.ops.triton_ops import LoRAKernelMeta, lora_shrink
from vllm.lora.ops.triton_ops.utils import _LORA_A_PTR_DICT
from vllm.utils.torch_utils import set_random_seed
DEVICE = "cuda"
@pytest.fixture(autouse=True)
def batch_invariant(monkeypatch):
monkeypatch.setattr(lora_utils, "is_batch_invariant", True)
monkeypatch.setattr(shrink_op, "is_batch_invariant", True)
lora_utils.get_lora_op_configs.cache_clear()
yield
lora_utils.get_lora_op_configs.cache_clear()
def _shrink(inputs, weights, mapping, scaling):
num_tokens = inputs.size(0)
meta = LoRAKernelMeta.make(weights[0].size(0), num_tokens, device=DEVICE)
meta.prepare_tensors(mapping)
out = torch.zeros(
(len(weights), num_tokens, weights[0].size(1)),
dtype=torch.float32,
device=DEVICE,
)
_LORA_A_PTR_DICT.clear()
args = meta.meta_args(token_nums=num_tokens, specialize_active_lora=False)
lora_shrink(inputs, weights, out, *args, scaling)
return out
@skip_if_not_cuda_alike
@pytest.mark.parametrize(
"hidden_size,rank,nslices,dtype",
[
(2560, 8, 3, torch.bfloat16),
(9728, 16, 1, torch.float16),
(4097, 64, 1, torch.bfloat16), # K tail block
(1024, 8, 1, torch.float16), # K < 8 * BLOCK_K: some splits are empty
],
)
def test_lora_shrink_matches_reference(hidden_size, rank, nslices, dtype):
set_random_seed(0)
inputs = torch.randn((33, hidden_size), dtype=dtype, device=DEVICE)
weights = [
torch.randn((2, rank, hidden_size), dtype=dtype, device=DEVICE)
for _ in range(nslices)
]
mapping = torch.randint(-1, 2, (33,), dtype=torch.int32, device=DEVICE)
out = _shrink(inputs, weights, mapping, 0.5).double().cpu()
ids, x = mapping.cpu(), inputs.double().cpu()
ref = torch.zeros_like(out)
for s, weight in enumerate(weights):
for lora_id in (0, 1):
rows = ids == lora_id
ref[s, rows] = 0.5 * x[rows] @ weight[lora_id].double().cpu().T
torch.testing.assert_close(out, ref, rtol=5e-3, atol=5e-3)
@skip_if_not_cuda_alike
@pytest.mark.parametrize("nslices", [1, 3])
def test_lora_shrink_invariant_across_batch_sizes(monkeypatch, nslices):
reduce_kernel, reduce_grids = shrink_op._lora_shrink_reduce_kernel, []
class RecordReduce:
def __getitem__(self, grid):
reduce_grids.append(grid)
return reduce_kernel[grid]
monkeypatch.setattr(shrink_op, "_lora_shrink_reduce_kernel", RecordReduce())
set_random_seed(0)
# K = 2560 is ten BLOCK_K=256 blocks, so all 8 splits contribute.
dtype, hidden_size = torch.bfloat16, 2560
weights = [
torch.randn((3, 16, hidden_size), dtype=dtype, device=DEVICE)
for _ in range(nslices)
]
target = torch.randn((1, hidden_size), dtype=dtype, device=DEVICE)
target_id = torch.zeros(1, dtype=torch.int32, device=DEVICE)
expected = _shrink(target, weights, target_id, 0.7)[:, 0]
for num_tokens in (31, 32, 33, 127, 128, 129):
inputs = torch.randn((num_tokens, hidden_size), dtype=dtype, device=DEVICE)
mapping = torch.randint(-1, 3, (num_tokens,), dtype=torch.int32, device=DEVICE)
pos = num_tokens // 2
inputs[pos], mapping[pos] = target[0], 0
out = _shrink(inputs, weights, mapping, 0.7)
assert torch.equal(out[:, pos], expected), num_tokens
assert len(reduce_grids) == 7
tests/v1/engine/test_admission_control.py
-2
View File
@@ -96,7 +96,6 @@ def _attach_prometheus_logger(
vllm_config.speculative_config = None
vllm_config.kv_transfer_config = None
vllm_config.ec_transfer_config = None
vllm_config.lora_config = None
llm.logger_manager = StatLoggerManager(
vllm_config=vllm_config,
engine_idxs=engine_indexes,
@@ -128,7 +127,6 @@ def _make_engine_request(request_id: str, n: int) -> EngineCoreRequest:
sampling_params=SamplingParams(n=n),
pooling_params=None,
arrival_time=0,
lora_request=None,
cache_salt=None,
data_parallel_rank=None,
)
tests/v1/spec_decode/test_dflash_profile.py
-3
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from contextlib import nullcontext
from types import SimpleNamespace
import pytest
@@ -47,8 +46,6 @@ def _run_dummy_batch(
speculator=speculator,
kv_connector=SimpleNamespace(set_disabled=lambda disabled: None),
is_first_pp_rank=True,
lora_config=None,
maybe_dummy_run_with_lora=lambda *args, **kwargs: nullcontext(),
execute_model=execute_model,
)
tests/v1/spec_decode/test_dynamic_sd_cug.py
-10
View File
@@ -137,7 +137,6 @@ def test_dynamic_sd_full_cudagraph_covers_all_uniform_decode_shapes(monkeypatch)
num_reqs=num_reqs,
num_tokens=num_tokens,
uniform_token_count=uniform_tok_count,
num_active_loras=0,
)
# With DSD enabled, MRv2 should have captured a FULL candidate for
@@ -146,7 +145,6 @@ def test_dynamic_sd_full_cudagraph_covers_all_uniform_decode_shapes(monkeypatch)
assert desc.uniform_token_count == max_query_len
assert desc.num_tokens == num_tokens
assert desc.num_reqs == num_reqs
assert desc.num_active_loras == 0
def test_dynamic_sd_cudagraphs_use_clamped_query_length(monkeypatch):
@@ -216,14 +214,12 @@ def test_dynamic_sd_non_uniform_batch_falls_back_to_piecewise(monkeypatch):
num_reqs=2,
num_tokens=3,
uniform_token_count=None,
num_active_loras=0,
)
assert desc.cg_mode == CUDAGraphMode.PIECEWISE
assert desc.uniform_token_count is None
assert desc.num_reqs is None
assert desc.num_tokens == 3
assert desc.num_active_loras == 0
def test_prompt_chunks_shaped_like_spec_decode_miss_the_full_graph(monkeypatch):
@@ -276,7 +272,6 @@ def test_prompt_chunks_shaped_like_spec_decode_miss_the_full_graph(monkeypatch):
num_reqs=num_reqs,
num_tokens=num_tokens,
uniform_token_count=uniform_tok_count,
num_active_loras=0,
)
assert desc.cg_mode == CUDAGraphMode.PIECEWISE
assert desc.uniform_token_count is None
@@ -290,7 +285,6 @@ def test_prompt_chunks_shaped_like_spec_decode_miss_the_full_graph(monkeypatch):
num_reqs=num_reqs,
num_tokens=num_tokens,
uniform_token_count=uniform_tok_count,
num_active_loras=0,
)
assert desc.cg_mode == CUDAGraphMode.FULL
assert desc.uniform_token_count == decode_query_len
@@ -344,14 +338,12 @@ def test_basic_sd_does_not_capture_shorter_full_decode_shapes(monkeypatch):
num_reqs=num_reqs,
num_tokens=num_tokens,
uniform_token_count=uniform_tok_count,
num_active_loras=0,
)
assert desc.cg_mode == CUDAGraphMode.PIECEWISE
assert desc.uniform_token_count is None
assert desc.num_tokens == num_tokens
assert desc.num_reqs is None
assert desc.num_active_loras == 0
def test_dynamic_sd_only_captures_scheduled_query_lengths(monkeypatch):
@@ -409,7 +401,6 @@ def test_dynamic_sd_only_captures_scheduled_query_lengths(monkeypatch):
num_reqs=num_reqs,
num_tokens=num_tokens,
uniform_token_count=uniform_tok_count,
num_active_loras=0,
)
if max_query_len in scheduled_query_lens:
@@ -425,4 +416,3 @@ def test_dynamic_sd_only_captures_scheduled_query_lengths(monkeypatch):
assert desc.uniform_token_count is None
assert desc.num_tokens == num_tokens
assert desc.num_reqs is None
assert desc.num_active_loras == 0
tests/v1/streaming_input/test_gpu_model_runner_v2_streaming.py
-5
View File
@@ -35,7 +35,6 @@ def mock_model_runner_with_req_states():
runner.encoder_cache = None
runner.model_state = Mock()
runner.block_tables = Mock()
runner.lora_state = Mock()
runner.pp_handler = None
runner.sampler = None
runner.prompt_logprobs_worker = None
@@ -91,7 +90,6 @@ def test_e2e_streaming_request_update_basic_flow(
pooling_params=None,
block_ids=([0],),
num_computed_tokens=3,
lora_request=None,
)
runner.add_requests(_make_scheduler_output([initial_req_data]))
assert req_id in req_states.req_id_to_index
@@ -109,7 +107,6 @@ def test_e2e_streaming_request_update_basic_flow(
pooling_params=None,
block_ids=([0, 1],),
num_computed_tokens=4, # 3 original prompt + 1 intermediate output
lora_request=None,
)
runner.add_requests(_make_scheduler_output([updated_req_data]))
@@ -163,7 +160,6 @@ def test_e2e_streaming_with_multimodal_features(
pooling_params=None,
block_ids=([0],),
num_computed_tokens=14,
lora_request=None,
)
runner.add_requests(_make_scheduler_output([initial_req_data]))
assert req_id in req_states.req_id_to_index
@@ -185,7 +181,6 @@ def test_e2e_streaming_with_multimodal_features(
pooling_params=None,
block_ids=([0, 1],),
num_computed_tokens=14,
lora_request=None,
)
runner.add_requests(_make_scheduler_output([updated_req_data]))
tests/v1/worker/test_gpu_model_runner_v2.py
-2
View File
@@ -270,8 +270,6 @@ def _make_capture_runner(captured: bool) -> GPUModelRunner:
needs_capture=lambda: captured,
capture=lambda *args, **kwargs: None,
)
runner.lora_config = None
runner.maybe_setup_dummy_loras = lambda _cfg: contextlib.nullcontext()
runner.speculator = None
runner.adaptive_verification = None
runner.model = None
tests/v1/worker/test_gpu_model_runner_v2_cudagraph_profiling.py
-4
View File
@@ -363,8 +363,6 @@ def test_profile_cudagraph_memory_frees_throwaway_pool(monkeypatch):
runner.kv_caches = []
runner.attn_groups = []
runner.kv_cache_config = SimpleNamespace()
runner.lora_config = None
runner.maybe_remove_all_loras = lambda _: None
allocation_bytes = 64 << 20
memory: dict[str, int] = {}
@@ -432,8 +430,6 @@ def test_teardown_profiling_state_clears_mamba_align_metadata(monkeypatch):
runner.attn_groups = []
runner.kv_cache_config = SimpleNamespace()
runner.cudagraph_manager = object()
runner.lora_config = None
runner.maybe_remove_all_loras = lambda _: None
monkeypatch.setattr(cgu.torch.accelerator, "synchronize", lambda: None)
monkeypatch.setattr(cgu.torch.accelerator, "empty_cache", lambda: None)
tests/v1/worker/test_gpu_model_runner_v2_eplb.py
-1
View File
@@ -81,7 +81,6 @@ def _make_runner(**overrides: Any) -> Any:
load_config=runner.load_config,
model_config=runner.model_config,
)
runner.lora_config = None
runner.use_aux_hidden_state_outputs = False
runner.speculative_config = None
runner.speculator = None
tests/v1/worker/test_gpu_worker_weight_transfer.py
+7 -25
View File
@@ -8,12 +8,9 @@ session is active. These tests verify that delegation and the session guard.
"""
import pytest
import torch
import torch.nn as nn
from vllm.config import ParallelConfig, VllmConfig, get_current_vllm_config
from vllm.lora.layers import BaseLayerWithLoRA
from vllm.model_executor.model_loader.reload.inplace import get_parameter_for_reload
from vllm.v1.worker.gpu_worker import Worker
@@ -57,13 +54,13 @@ class _RecordingEngine:
class _RecordingModelRunner:
def __init__(self) -> None:
self.seen_config: VllmConfig | None = None
self.reset_lora_calls = 0
self.refresh_draft_weights_calls = 0
def reload_weights(self) -> None:
self.seen_config = get_current_vllm_config()
def reset_lora_state(self) -> None:
self.reset_lora_calls += 1
def refresh_draft_weights(self) -> None:
self.refresh_draft_weights_calls += 1
def get_draft_model(self) -> nn.Module | None:
return None
@@ -75,6 +72,7 @@ def _make_worker(engine: _RecordingEngine | None) -> Worker:
worker.weight_transfer_engine = engine
worker._weight_update_active = False
worker._weight_update_is_draft = False
worker.use_v2_model_runner = False
worker.model_runner = _RecordingModelRunner()
return worker
@@ -108,22 +106,6 @@ def test_shutdown_weight_transfer_engine_preserves_inference(enabled, active):
assert worker.model_runner is model_runner
def test_reload_parameter_lookup_preserves_lora_module_names():
base_layer = nn.Module()
qweight = nn.Parameter(torch.ones(1))
base_layer.register_parameter("qweight", qweight)
wrapper = BaseLayerWithLoRA()
wrapper.base_layer = base_layer
model = nn.Module()
model.proj = wrapper
named_parameters = dict(model.named_parameters())
assert set(named_parameters) == {"proj.base_layer.qweight"}
assert named_parameters["proj.base_layer.qweight"] is qweight
assert model.get_parameter("proj.base_layer.qweight") is qweight
assert get_parameter_for_reload(model, "proj.qweight") is qweight
def test_start_update_finish_delegates_to_engine():
engine = _RecordingEngine()
worker = _make_worker(engine)
@@ -141,7 +123,7 @@ def test_start_update_finish_delegates_to_engine():
assert engine.reset_count == 1
assert worker._weight_update_active is False
assert engine.seen_configs == [worker.vllm_config] * 3
assert worker.model_runner.reset_lora_calls == 1
assert worker.model_runner.refresh_draft_weights_calls == 0
@pytest.mark.parametrize(
@@ -194,7 +176,7 @@ def test_rank_local_update_uses_data_parallel_index_after_reconfigure():
assert worker._weight_update_active is True
def test_finish_draft_session_keeps_lora_state():
def test_finish_draft_session_does_not_refresh_draft_weights():
engine = _RecordingEngine()
engine.supports_draft_weight_update = True
worker = _make_worker(engine)
@@ -203,7 +185,7 @@ def test_finish_draft_session_keeps_lora_state():
Worker.start_draft_weight_update(worker)
Worker.finish_weight_update(worker)
assert worker.model_runner.reset_lora_calls == 0
assert worker.model_runner.refresh_draft_weights_calls == 0
def test_double_start_raises():
vllm/_custom_ops.py
-32
View File
@@ -2084,38 +2084,6 @@ def batched_moe_align_block_size(
)
def moe_lora_align_block_size(
topk_ids: torch.Tensor,
token_lora_mapping: torch.Tensor,
num_experts: int,
block_size: int,
max_loras: int,
max_num_tokens_padded: int,
max_num_m_blocks: int,
sorted_token_ids: torch.Tensor,
experts_ids: torch.Tensor,
num_tokens_post_pad: torch.Tensor,
adapter_enabled: torch.Tensor,
lora_ids: torch.Tensor,
expert_map: torch.Tensor | None = None,
) -> None:
torch.ops._moe_C.moe_lora_align_block_size(
topk_ids,
token_lora_mapping,
num_experts,
block_size,
max_loras,
max_num_tokens_padded,
max_num_m_blocks,
sorted_token_ids,
experts_ids,
num_tokens_post_pad,
adapter_enabled,
lora_ids,
expert_map,
)
def moe_wna16_gemm(
input: torch.Tensor,
output: torch.Tensor,
vllm/config/__init__.py
-7
View File
@@ -21,7 +21,6 @@ from vllm.config.kv_events import KVEventsConfig
from vllm.config.kv_transfer import KVTransferConfig
from vllm.config.load import LoadConfig
from vllm.config.logging import LoggingConfig
from vllm.config.lora import LoRAConfig
from vllm.config.mamba import MambaConfig
from vllm.config.model import (
ModelConfig,
@@ -43,7 +42,6 @@ from vllm.config.profiler import ProfilerConfig
from vllm.config.reasoning import ReasoningConfig
from vllm.config.scheduler import SchedulerConfig
from vllm.config.speculative import SpeculativeConfig
from vllm.config.speech_to_text import SpeechToTextConfig, SpeechToTextParams
from vllm.config.structured_outputs import StructuredOutputsConfig
from vllm.config.utils import (
ConfigType,
@@ -100,8 +98,6 @@ __all__ = [
"LoadConfig",
# From vllm.config.logging
"LoggingConfig",
# From vllm.config.lora
"LoRAConfig",
# From vllm.config.mamba
"MambaConfig",
# From vllm.config.model
@@ -129,9 +125,6 @@ __all__ = [
"SchedulerConfig",
# From vllm.config.speculative
"SpeculativeConfig",
# From vllm.config.speech_to_text
"SpeechToTextConfig",
"SpeechToTextParams",
# From vllm.config.structured_outputs
"StructuredOutputsConfig",
# From vllm.config.profiler
vllm/config/compilation.py
+2 -2
View File
@@ -559,7 +559,7 @@ class CompilationConfig:
cudagraph_decoder_replay: bool = True
"""Run the decoder replay layers of YOCO models (e.g. DeepSeek-V4.1) in CUDA
graphs of their own, and trim them in PIECEWISE graph steps. Requires
breakable PIECEWISE graphs; off under LoRA, prompt embeddings and non-first
breakable PIECEWISE graphs; off with prompt embeddings and non-first
PP ranks."""
decoder_replay_cudagraph_capture_sizes: list[int] = field(default_factory=list)
@@ -666,7 +666,7 @@ class CompilationConfig:
for all cases, incurring the overhead of running LoRA ops even when no
adapters are active. Setting this to True will remove this overhead at the
cost of increased startup time and slightly higher memory usage.
When `enable_lora` is False, this option has no effect.
When LoRA is not enabled, this option has no effect.
"""
use_inductor_graph_partition: bool = None # type: ignore[assignment]
vllm/config/lora.py
-151
View File
@@ -1,151 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
from typing import TYPE_CHECKING, Any, Literal, Self
import torch
from pydantic import ConfigDict, Field, model_validator
from vllm import envs
from vllm.config.utils import config
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.utils.hashing import safe_hash
if TYPE_CHECKING:
from vllm.config import ModelConfig
from vllm.config.cache import CacheConfig
else:
ModelConfig = Any
CacheConfig = Any
logger = init_logger(__name__)
LoRADType = Literal["auto", "float16", "bfloat16"]
MaxLoRARanks = Literal[1, 8, 16, 32, 64, 128, 256, 320, 512]
@config(config=ConfigDict(arbitrary_types_allowed=True))
class LoRAConfig:
"""Configuration for LoRA."""
max_lora_rank: MaxLoRARanks = 16
"""Max LoRA rank."""
max_loras: int = Field(default=1, ge=1)
"""Max number of LoRAs in a single batch."""
fully_sharded_loras: bool = False
"""By default, only half of the LoRA computation is sharded with tensor
parallelism. Enabling this will use the fully sharded layers. At high
sequence length, max rank or tensor parallel size, this is likely faster.
"""
max_cpu_loras: int | None = None
"""Maximum number of LoRAs to store in CPU memory. Must be >= than
`max_loras`."""
lora_dtype: torch.dtype | LoRADType = "auto"
"""Data type for LoRA. If auto, will default to base model dtype."""
target_modules: list[str] | None = None
"""Restrict LoRA to specific module suffixes (e.g., ["o_proj", "qkv_proj"]).
If None, all supported LoRA modules are used. This allows deployment-time
control over which modules have LoRA applied, useful for performance tuning."""
default_mm_loras: dict[str, str] | None = None
"""Dictionary mapping specific modalities to LoRA model paths; this field
is only applicable to multimodal models and should be leveraged when a
model always expects a LoRA to be active when a given modality is present.
Note that currently, if a request provides multiple additional
modalities, each of which have their own LoRA, we do NOT apply
default_mm_loras because we currently only support one lora adapter
per prompt. When run in offline mode, the lora IDs for n modalities
will be automatically assigned to 1-n with the names of the modalities
in alphabetic order."""
enable_tower_connector_lora: bool = False
"""If `True`, LoRA support for the tower (vision encoder) and connector
of multimodal models will be enabled. This is an experimental feature and
currently only supports some MM models such as the Qwen VL series. The default
is False."""
specialize_active_lora: bool = False
"""Whether to construct lora kernel grid by the number of active LoRA adapters.
When set to True, separate cuda graphs will be captured for different counts
of active LoRAs (powers of 2 up to max_loras), which can improve performance
for variable LoRA usage patterns at the cost of increased startup time and
memory usage. Only takes effect when cudagraph_specialize_lora is True.
"""
enable_mixed_moe_lora_format: bool = False
"""If True, force the engine to use the universal 2D MoE LoRA wrapper
(`FusedMoEWithLoRA`) regardless of the model's `is_3d_moe_weight` flag, so
that 2D-format and 3D-format MoE LoRA adapters can be served in the same
deployment. Only meaningful for MoE models; ignored otherwise. Default False
keeps the existing model-driven behavior."""
enable_moe_shared_loras: bool = False
"""If True, load MoE expert adapters in the "shared-outer" layout, where the
gate/up (`w1`/`w3`) lora_A and the down (`w2`) lora_B are shared across all
experts (stored once with expert-dim 1) instead of per-expert. The shared
factors are broadcast to the expert count at kernel time. 3D-weight MoE
models use `FusedMoEWithLoRA` for this layout. Only meaningful for MoE
models whose adapters use this layout; ignored otherwise."""
max_lora_cls_labels: int | None = Field(default=None, ge=1)
"""Maximum output size for LoRA classification heads. Defaults to the
base classification head size."""
def compute_hash(self) -> str:
"""WARNING: Whenever a new field is added to this config,
ensure that it is included in the factors list if
it affects the computation graph.
Provide a hash that uniquely identifies all the configs
that affect the structure of the computation
graph from input ids/embeddings to the final hidden states,
excluding anything before input ids/embeddings and after
the final hidden states.
"""
factors: list[Any] = []
factors.append(self.max_lora_rank)
factors.append(self.max_loras)
factors.append(self.fully_sharded_loras)
factors.append(self.lora_dtype)
factors.append(self.enable_tower_connector_lora)
factors.append(self.enable_mixed_moe_lora_format)
factors.append(self.enable_moe_shared_loras)
factors.append(self.max_lora_cls_labels)
# target_modules affects which modules get LoRA applied
factors.append(
tuple(sorted(self.target_modules)) if self.target_modules else None
)
hash_str = safe_hash(str(factors).encode(), usedforsecurity=False).hexdigest()
return hash_str
@model_validator(mode="after")
def _validate_lora_config(self) -> Self:
if self.max_cpu_loras is None:
self.max_cpu_loras = self.max_loras
elif self.max_cpu_loras < self.max_loras:
raise ValueError(
f"max_cpu_loras ({self.max_cpu_loras}) must be >= "
f"max_loras ({self.max_loras})."
)
if envs.VLLM_LORA_ENABLE_DUAL_STREAM and not current_platform.is_cuda_alike():
raise ValueError("Dual CUDA streams are only supported on CUDA platforms.")
if envs.VLLM_LORA_ENABLE_DUAL_STREAM and self.fully_sharded_loras:
logger.warning_once(
"fully_sharded_loras isn't compatible with "
"VLLM_LORA_ENABLE_DUAL_STREAM, set VLLM_LORA_ENABLE_DUAL_STREAM=False"
)
envs.VLLM_LORA_ENABLE_DUAL_STREAM = False
return self
def verify_with_model_config(self, model_config: ModelConfig):
if self.lora_dtype in (None, "auto"):
self.lora_dtype = model_config.dtype
elif isinstance(self.lora_dtype, str):
self.lora_dtype = getattr(torch, self.lora_dtype)
architectures = getattr(model_config, "architectures", None) or []
is_inkling = any("Inkling" in arch for arch in architectures)
if is_inkling and os.environ.get("INKLING_MULTIMEM_AR", "1") != "0":
raise ValueError(
"Inkling LoRA requires INKLING_MULTIMEM_AR=0: the Lamport "
"fused-collective path bypasses the LoRA-wrapped wo_ud and dense "
"down_proj layers on decode-sized batches, silently dropping "
"their LoRA. Set INKLING_MULTIMEM_AR=0."
)
vllm/config/scheduler.py
+5 -9
View File
@@ -268,15 +268,11 @@ class SchedulerConfig:
"""
factors: list[Any] = []
# max_num_batched_tokens need to be included in the hash due
# to two reasons:
# 1. LoRA creates static buffers based on max_num_batched_tokens.
# The tensor sizes and strides get captured in the torch.compile
# graph explicitly.
# 2. Inductor decides whether using 32-bit or 64-bit indexing integer
# based on the data sizes. `max_num_batched_tokens` has an
# impact on that. For more details, please check
# https://github.com/vllm-project/vllm/issues/29585
# max_num_batched_tokens need to be included in the hash because
# inductor decides whether using 32-bit or 64-bit indexing integer
# based on the data sizes. `max_num_batched_tokens` has an
# impact on that. For more details, please check
# https://github.com/vllm-project/vllm/issues/29585
factors.append(self.max_num_batched_tokens)
# PLE and other model components allocate static per-request buffers.
vllm/config/speech_to_text.py
-85
View File
@@ -1,85 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
from dataclasses import dataclass
from typing import TYPE_CHECKING
from vllm.config.utils import config
if TYPE_CHECKING:
import numpy as np
from vllm.config.model import ModelConfig
@dataclass
class SpeechToTextParams:
"""All parameters consumed by ``get_generation_prompt()``.
``TranscriptionRequest.build_stt_params()`` constructs this object,
mapping API-level fields into typed attributes. Models only receive
this object, so new parameters can be added here without changing the
``get_generation_prompt`` signature.
"""
audio: np.ndarray
"""Resampled audio waveform for a single chunk."""
stt_config: SpeechToTextConfig
"""Server-level speech-to-text configuration."""
model_config: ModelConfig
"""Model configuration."""
language: str | None = None
"""ISO 639-1 language code (validated / auto-detected)."""
hotwords: str | None = None
"""
hotwords refers to a list of important words or phrases that the model
should pay extra attention to during transcription.
"""
task_type: str = "transcribe"
"""``"transcribe"`` or ``"translate"``."""
request_prompt: str = ""
"""Optional text prompt to guide the model."""
to_language: str | None = None
"""Target language for translation (model-dependent)."""
@config
class SpeechToTextConfig:
"""Configuration for speech-to-text models."""
sample_rate: float = 16_000
"""Sample rate (Hz) to resample input audio to. Most speech models expect
16kHz audio input. The input audio will be automatically resampled to this
rate before processing."""
max_audio_clip_s: int | None = 30
"""Maximum duration in seconds for a single audio clip without chunking.
Audio longer than this will be split into smaller chunks if
`allow_audio_chunking` evaluates to True, otherwise it will be rejected.
`None` means audio duration can be unlimited and won't be chunked."""
overlap_chunk_second: int = 1
"""Overlap duration in seconds between consecutive audio chunks when
splitting long audio. This helps maintain context across chunk boundaries
and improves transcription quality at split points."""
min_energy_split_window_size: int | None = 1600
"""Window size in samples for finding low-energy (quiet) regions to split
audio chunks. The algorithm looks for the quietest moment within this
window to minimize cutting through speech. Default 1600 samples ≈ 100ms
at 16kHz. If None, no chunking will be done."""
@property
def allow_audio_chunking(self) -> bool:
return (
self.min_energy_split_window_size is not None
and self.max_audio_clip_s is not None
)
vllm/config/vllm.py
-18
View File
@@ -41,7 +41,6 @@ from .kv_events import KVEventsConfig
from .kv_transfer import KVTransferConfig
from .load import LoadConfig
from .logging import LoggingConfig
from .lora import LoRAConfig
from .mamba import MambaBackendEnum, MambaConfig
from .model import ModelConfig
from .observability import ObservabilityConfig
@@ -379,8 +378,6 @@ class VllmConfig:
"""Mamba configuration."""
kernel_config: KernelConfig = Field(default_factory=KernelConfig)
"""Kernel configuration."""
lora_config: LoRAConfig | None = None
"""LoRA configuration."""
speculative_config: SpeculativeConfig | None = None
"""Speculative decoding configuration."""
watermark_config: WatermarkConfig | None = None
@@ -520,10 +517,6 @@ class VllmConfig:
if self.engram_config is not None
else "None"
)
if self.lora_config:
vllm_factors.append(self.lora_config.compute_hash())
else:
vllm_factors.append("None")
if self.speculative_config:
vllm_factors.append(self.speculative_config.compute_hash())
else:
@@ -1553,8 +1546,6 @@ class VllmConfig:
"PD with decode_context_parallel_size > 1 is only "
"supported for MLA models."
)
if self.lora_config is not None:
self.lora_config.verify_with_model_config(self.model_config)
if (
self.mamba_config.enable_stochastic_rounding
@@ -3093,13 +3084,6 @@ class VllmConfig:
if not spec_config or not spec_config.enable_adaptive_verification:
return
if self.lora_config is not None:
# The per-token LoRA mapping is built from CPU placeholder boundaries,
# while the trimmed batch's true boundaries are decided on the GPU.
raise ValueError(
"Adaptive verification is not currently compatible with LoRA"
)
if not self.compilation_config.cudagraph_mode.has_full_cudagraphs():
raise ValueError(
"Adaptive verification requires full CUDA graphs. Use cudagraph "
@@ -3189,8 +3173,6 @@ class VllmConfig:
model_config = self.model_config
parallel_config = self.parallel_config
if self.lora_config is not None:
unsupported.append("dual batch overlap with LoRA")
if self.speculative_config is not None:
unsupported.append("dual batch overlap with speculative decoding")
if parallel_config.pipeline_parallel_size > 1:
vllm/distributed/kv_events.py
+4 -12
View File
@@ -59,20 +59,13 @@ class BlockStored(KVCacheEvent):
token_ids: list[int]
block_size: int
"""Deprecated: use `lora_name` for KV block key hash.
Retained for backward compatibility.
"""
lora_id: int | None
medium: str | None
lora_name: str | None
"""Extra keys used in block hash computation, one entry per block in
block_hashes. Each entry contains, in order, the LoRA name (if
`lora_name` is set), `(mm_identifier, offset)` pairs, the cache_salt
string and prompt embedding digests for that specific block. These are
published untagged; the block hash itself uses each value tagged with
its source, e.g. `("lora", name)` or `("mm", mm_identifier, offset)`.
block_hashes. Each entry contains, in order, the `(mm_identifier, offset)`
pairs, the cache_salt string and prompt embedding digests for that
specific block. These are published untagged; the block hash itself uses
each value tagged with its source, e.g. `("mm", mm_identifier, offset)`.
"""
extra_keys: list[tuple[Any, ...] | None] | None = None
@@ -100,7 +93,6 @@ class BlockStored(KVCacheEvent):
self.parent_block_hash,
tuple(self.token_ids),
self.block_size,
self.lora_id,
self.medium,
tuple(self.extra_keys) if self.extra_keys else None,
self.group_idx,
vllm/distributed/kv_transfer/README.md
-29
View File
@@ -1,29 +0,0 @@
# Distributed KV cache transfer
This folder implements distributed KV cache transfer across vLLM instances.
Currently the main use case is for disaggregated prefilling.
## Abstractions
The KV cache transfer contains three layer of abstractions:
- KV pipe: a FIFO pipe for torch.tensor transmission. Key APIs: `send_tensor` and `recv_tensor`.
- KV lookup buffer: a lookup buffer for KV caches. Key: the tokens, value: the KV caches (and/or hidden states). Key APIs: `insert` and `drop_select` (similar to SQL semantics).
- KV connector: a connector that connects the KV pipe and KV lookup buffer to vLLM. Key APIs: `send_kv_caches_and_hidden_states` and `recv_kv_caches_and_hidden_states`.
Why we need KV lookup buffer: FIFO pipe itself is not enough as prefill vLLM worker may process requests in a different order compared to decode vLLM worker. Say the QPS is really high, prefill worker may handle requests in order A -> B -> C, but the decode worker may process request C first. This is not the case that can be naturally handled by FIFO pipe, so we provide KV lookup buffer to help translate a FIFO pipe to a lookup buffer.
NOTE: KV pipe layer is bypassable: you can skip this layer if your distributed
communication service already supports key-value-based lookup (like redis or
RDMA database).
NOTE: If you want to not only transfer KV caches, but adjust the model execution flow of vLLM as well (for example, allow vLLM to receive KV caches on some tokens and do prefill on the remaining tokens), you can bypass both KV pipe layer and KV lookup buffer layer, and directly implement on KV connector layer. Bear in mind that as vLLM's model input is constantly changing, this implementation will likely be broken when vLLM has new updates.
## Disaggregated prefilling
Example usage can be found in [this directory](../../../examples/disaggregated).
Here is the diagram of how we run disaggregated prefilling.
![Disaggregated prefill workflow](./disagg_prefill_workflow.jpg)
vllm/distributed/kv_transfer/disagg_prefill_workflow.jpg
BIN
View File
Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 139 KiB

vllm/distributed/kv_transfer/kv_connector/factory.py
-6
View File
@@ -161,9 +161,3 @@ KVConnectorFactory.register_connector(
"vllm.distributed.kv_transfer.kv_connector.v1.offloading_connector",
"OffloadingConnector",
)
KVConnectorFactory.register_connector(
"DecodeBenchConnector",
"vllm.distributed.kv_transfer.kv_connector.v1.decode_bench_connector",
"DecodeBenchConnector",
)
vllm/distributed/kv_transfer/kv_connector/v1/__init__.py
-4
View File
@@ -6,14 +6,10 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
SupportsHMA,
supports_hma,
)
from vllm.distributed.kv_transfer.kv_connector.v1.decode_bench_connector import ( # noqa: E501
DecodeBenchConnector,
)
__all__ = [
"KVConnectorRole",
"KVConnectorBase_V1",
"supports_hma",
"SupportsHMA",
"DecodeBenchConnector",
]
vllm/distributed/kv_transfer/kv_connector/v1/decode_bench_connector.py
-734
View File
@@ -1,734 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""DecodeBenchConnector: A KV Connector for decode instance performance testing.
This connector emulates a prefill-decode disaggregated setting by filling
the KV cache with dummy values, allowing measurement of decoder performance
under larger input sequence lengths (ISL) in resource-limited environments.
Usage:
To use this connector for benchmarking, configure it in the kv_transfer_config:
Example:
vllm serve <model> --kv-transfer-config '{
"kv_connector": "DecodeBenchConnector",
"kv_role": "kv_both",
"kv_connector_extra_config": {
"fill_mean": 0.015,
"fill_std": 0.0
}
}'
Then run your benchmark with desired input/output lengths:
vllm bench serve --base-url http://127.0.0.1:8000 --model <model> \\
--dataset-name random --random-input-len 40000 \\
--random-output-len 100 --max-concurrency 10
Configuration options (via kv_connector_extra_config):
- fill_mean (float): Mean value for random normal fill (default: 0.015)
- fill_std (float): Standard deviation for random fill (default: 0.0)
Set to 0 for constant values, >0 for random sampling
- startup_fill (bool): Fill the whole KV cache once at startup instead
of filling each new request's blocks in the step that schedules it
(default: False). Circular-buffer caches are still zeroed per
request, and attention caches whose new blocks the engine zeroes
before use (hybrid Mamba or mixed-precision KV caches) are still
filled per request. The fill is not redone after sleep mode
discards the KV cache.
The dummy KV cache is only meant for performance measurement; outputs are
not meaningful for accuracy evaluation.
"""
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any
import torch
from vllm.distributed.kv_transfer.kv_connector.v1 import (
KVConnectorBase_V1,
KVConnectorRole,
)
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorMetadata,
SupportsHMA,
)
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.utils.math_utils import cdiv
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.core.kv_cache_utils import (
resolve_dcp_kv_block_size,
)
from vllm.v1.kv_cache_interface import (
AttentionSpec,
CircularBufferSpec,
KVCacheSpec,
KVQuantMode,
UniformTypeKVCacheSpecs,
iter_layer_specs,
)
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.forward_context import ForwardContext
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.request import Request
logger = init_logger(__name__)
def _get_fp8_dtype(spec: KVCacheSpec, cache_dtype: str) -> torch.dtype | None:
"""The fp8 dtype of a KV cache that stores plain fp8 values as uint8.
Returns None for other layouts, including packed ones that embed scales
or other metadata in the uint8 page.
"""
if (
not isinstance(spec, AttentionSpec)
or spec.dtype != torch.uint8
or spec.kv_quant_mode != KVQuantMode.FP8_PER_TENSOR
or spec.state_content_bytes is not None
):
return None
cache_dtype = getattr(spec, "cache_dtype_str", None) or cache_dtype
if cache_dtype in ("fp8", "fp8_e4m3"):
return current_platform.fp8_dtype()
if cache_dtype == "fp8_e5m2":
return torch.float8_e5m2
return None
def _get_startup_fill_group_ids(kv_cache_config: "KVCacheConfig") -> set[int]:
"""KV cache groups that startup fill covers.
Circular buffers are zero-filled per request. Attention groups whose new
blocks the engine zeroes before use are filled per request, since the
zeroing would wipe the startup fill.
"""
group_ids = set()
for group_idx, group in enumerate(kv_cache_config.kv_cache_groups):
spec = group.kv_cache_spec
if any(isinstance(s, CircularBufferSpec) for s in iter_layer_specs(spec)):
continue
if kv_cache_config.needs_kv_cache_zeroing and isinstance(spec, AttentionSpec):
continue
group_ids.add(group_idx)
return group_ids
@dataclass
class DecodeBenchConnectorMetadata(KVConnectorMetadata):
"""Metadata for DecodeBenchConnector.
Contains information about which requests need their KV cache filled
with dummy values for benchmarking purposes.
"""
# request_id -> (block_ids_per_group, num_tokens_to_fill)
# block_ids_per_group is a tuple of lists, one per KV cache group
# One group: ([1, 2, 3],)
# Multiple groups: ([1, 2], [5, 6])
reqs_to_fill: dict[str, tuple[tuple[list[int], ...], int]]
class DecodeBenchConnector(KVConnectorBase_V1, SupportsHMA):
"""A KV Connector for decode instance performance testing.
This connector fills the KV cache with dummy values to emulate a
prefill-decode disaggregated setting, enabling performance testing of the
decoder with larger input sequence lengths.
"""
def __init__(
self,
vllm_config: "VllmConfig",
role: KVConnectorRole,
kv_cache_config: "KVCacheConfig",
):
super().__init__(vllm_config, role, kv_cache_config)
self.connector_scheduler: DecodeBenchConnectorScheduler | None = None
self.connector_worker: DecodeBenchConnectorWorker | None = None
if role == KVConnectorRole.SCHEDULER:
self.connector_scheduler = DecodeBenchConnectorScheduler(
vllm_config, kv_cache_config
)
elif role == KVConnectorRole.WORKER:
self.connector_worker = DecodeBenchConnectorWorker(
vllm_config, kv_cache_config
)
# ==============================
# Worker-side methods
# ==============================
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
assert self.connector_worker is not None
self.connector_worker.register_kv_caches(kv_caches)
def start_load_kv(self, forward_context: "ForwardContext", **kwargs: Any) -> None:
assert self.connector_worker is not None
assert isinstance(self._connector_metadata, DecodeBenchConnectorMetadata)
self.connector_worker.start_fill_kv(self._connector_metadata)
def wait_for_layer_load(self, layer_name: str) -> None:
# All operations are synchronous, so nothing to wait for
pass
def save_kv_layer(
self,
layer_name: str,
kv_layer: torch.Tensor,
attn_metadata: AttentionMetadata,
**kwargs: Any,
) -> None:
# This connector doesn't save KV cache (benchmarking only)
pass
def wait_for_save(self):
# This connector doesn't save KV cache (benchmarking only)
pass
# ==============================
# Scheduler-side methods
# ==============================
def get_num_new_matched_tokens(
self,
request: "Request",
num_computed_tokens: int,
) -> tuple[int | None, bool]:
assert self.connector_scheduler is not None
return self.connector_scheduler.get_num_new_matched_tokens(
request, num_computed_tokens
)
def update_state_after_alloc(
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
):
assert self.connector_scheduler is not None
return self.connector_scheduler.update_state_after_alloc(
request, blocks, num_external_tokens
)
def build_connector_meta(
self, scheduler_output: "SchedulerOutput"
) -> KVConnectorMetadata:
assert self.connector_scheduler is not None
return self.connector_scheduler.build_connector_meta(scheduler_output)
def request_finished(
self,
request: "Request",
block_ids: list[int],
) -> tuple[bool, dict[str, Any] | None]:
assert self.connector_scheduler is not None
self.connector_scheduler.request_finished(request)
return False, None
def request_finished_all_groups(
self,
request: "Request",
block_ids: tuple[list[int], ...],
) -> tuple[bool, dict[str, Any] | None]:
# HMA-enabled path: same cleanup as the single-group variant since
# this connector owns no external state per block.
assert self.connector_scheduler is not None
self.connector_scheduler.request_finished(request)
return False, None
class DecodeBenchConnectorScheduler:
"""Scheduler-side implementation for DecodeBenchConnector."""
def __init__(self, vllm_config: "VllmConfig", kv_cache_config: "KVCacheConfig"):
kv_transfer_config = vllm_config.kv_transfer_config
assert kv_transfer_config is not None
self.startup_fill = kv_transfer_config.get_from_extra_config(
"startup_fill", False
)
self._startup_fill_group_ids = (
_get_startup_fill_group_ids(kv_cache_config) if self.startup_fill else set()
)
dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size
self.kv_cache_groups = kv_cache_config.kv_cache_groups
self.group_block_sizes = tuple(
resolve_dcp_kv_block_size(group.kv_cache_spec, dcp_world_size)
for group in self.kv_cache_groups
)
# Track which requests have already been filled
self._filled_requests: set[str] = set()
# Track pending fills for the current scheduler step
# request_id -> (block_ids_per_group, num_tokens_to_fill)
# Note: _pending_fills doesn't need explicit cleanup - it's cleared
# after build_connector_meta() is called in the same scheduler step
self._pending_fills: dict[str, tuple[tuple[list[int], ...], int]] = {}
def get_num_new_matched_tokens(
self,
request: "Request",
num_computed_tokens: int,
) -> tuple[int, bool]:
"""For new requests, return the number of tokens that should be filled
with dummy KV cache values.
Returns:
(num_tokens_to_fill, is_async)
- num_tokens_to_fill: number of uncomputed tokens minus 1
(we fill everything except the last token for decode)
- is_async: False (synchronous filling)
"""
req_id = request.request_id
# Only fill once per request on first scheduling
if req_id in self._filled_requests:
return 0, False
# Calculate how many tokens we need to fill
# Fill all uncomputed tokens except the last one (which will be decoded)
# This simulates having processed a long prefill
num_uncomputed_tokens = request.num_tokens - num_computed_tokens
num_tokens_to_fill = max(0, num_uncomputed_tokens - 1)
if num_tokens_to_fill == 0:
return 0, False
# Return False for synchronous operation - the fill is fast enough
# that async overhead isn't worth it
return num_tokens_to_fill, False
def update_state_after_alloc(
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
):
"""Called after blocks are allocated. Store the block IDs so we can
fill them with dummy values.
Supports both single- and multi-group KV cache configurations.
"""
req_id = request.request_id
if num_external_tokens == 0:
return
total_computed_tokens = request.num_tokens - 1
num_local_computed_tokens = total_computed_tokens - num_external_tokens
block_ids_per_group_list: list[list[int]] = []
for group_idx, (
group,
group_blocks,
group_block_size,
) in enumerate(
zip(
self.kv_cache_groups,
blocks.blocks,
self.group_block_sizes,
strict=True,
)
):
if group_idx in self._startup_fill_group_ids:
block_ids_per_group_list.append([])
continue
is_circular_buffer = all(
isinstance(spec, CircularBufferSpec)
for spec in iter_layer_specs(group.kv_cache_spec)
)
if not is_circular_buffer:
num_computed_blocks = cdiv(total_computed_tokens, group_block_size)
external_block_start = num_local_computed_tokens // group_block_size
assert (
0
<= external_block_start
<= num_computed_blocks
<= len(group_blocks)
), (
"DecodeBenchConnector block range exceeds allocated blocks: "
f"request={req_id}, group={group_idx}, "
f"range=[{external_block_start}, {num_computed_blocks}), "
f"allocated={len(group_blocks)}"
)
selected_blocks = group_blocks[external_block_start:num_computed_blocks]
else:
selected_blocks = group_blocks
block_ids = [
block.block_id for block in selected_blocks if not block.is_null
]
if not block_ids:
logger.warning(
"DecodeBenchConnector: No blocks selected for KV cache group "
"%d with %d external tokens for request %s",
group_idx,
num_external_tokens,
req_id,
)
block_ids_per_group_list.append(block_ids)
block_ids_per_group = tuple(block_ids_per_group_list)
self._filled_requests.add(req_id)
if self.startup_fill and not any(block_ids_per_group):
return
# Store the blocks to fill for all group. _pending_fills doesn't need cleanup
# as it's cleared after build_connector_meta
self._pending_fills[req_id] = (
block_ids_per_group,
num_external_tokens,
)
block_counts = tuple(len(group) for group in block_ids_per_group)
logger.debug(
"DecodeBenchConnector: Selected %d total blocks across %d KV cache "
"groups for request %s (per-group counts: %s)",
sum(block_counts),
len(blocks.blocks),
req_id,
", ".join(map(str, block_counts)),
)
def build_connector_meta(
self, scheduler_output: "SchedulerOutput"
) -> KVConnectorMetadata:
"""Build metadata containing information about which blocks to fill
with dummy KV values.
"""
meta = DecodeBenchConnectorMetadata(reqs_to_fill=self._pending_fills.copy())
# Clear pending fills after building metadata
self._pending_fills.clear()
return meta
def request_finished(self, request: "Request"):
"""Called when a request has finished. Clean up any state."""
self._filled_requests.discard(request.request_id)
class DecodeBenchConnectorWorker:
"""Worker-side implementation for DecodeBenchConnector."""
def __init__(self, vllm_config: "VllmConfig", kv_cache_config: "KVCacheConfig"):
# Get fill parameters from extra config
kv_transfer_config = vllm_config.kv_transfer_config
assert kv_transfer_config is not None
self.fill_mean = kv_transfer_config.get_from_extra_config("fill_mean", 0.015)
self.fill_std = kv_transfer_config.get_from_extra_config("fill_std", 0.0)
self.startup_fill = kv_transfer_config.get_from_extra_config(
"startup_fill", False
)
self._startup_fill_group_ids = (
_get_startup_fill_group_ids(kv_cache_config) if self.startup_fill else set()
)
# Will be populated via register_kv_caches
self.kv_caches: dict[str, torch.Tensor] | None = None
# Layers whose uint8 KV cache stores plain fp8 values
self._fp8_dtypes: dict[str, torch.dtype] = {}
for group in kv_cache_config.kv_cache_groups:
for layer_name in group.layer_names:
spec = group.kv_cache_spec
if isinstance(spec, UniformTypeKVCacheSpecs):
spec = spec.kv_cache_specs[layer_name]
fp8_dtype = _get_fp8_dtype(spec, vllm_config.cache_config.cache_dtype)
if fp8_dtype is not None:
self._fp8_dtypes[layer_name] = fp8_dtype
# Mapping from KV cache group index to list of layer names in that group
self.group_to_layers = {
group_idx: list(group.layer_names)
for group_idx, group in enumerate(kv_cache_config.kv_cache_groups)
}
self._zero_fill_group_ids = {
group_idx
for group_idx, group in enumerate(kv_cache_config.kv_cache_groups)
if any(
isinstance(spec, CircularBufferSpec)
for spec in iter_layer_specs(group.kv_cache_spec)
)
}
num_zeroed_groups = len(
self.group_to_layers.keys()
- self._startup_fill_group_ids
- self._zero_fill_group_ids
)
if self.startup_fill and num_zeroed_groups:
logger.info(
"DecodeBenchConnector: the engine zeroes new blocks of %d KV "
"cache groups before use, so they are still filled per request.",
num_zeroed_groups,
)
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
"""Store references to the KV cache tensors."""
self.kv_caches = kv_caches
logger.debug(
"DecodeBenchConnector: Registered %d KV cache layers",
len(kv_caches),
)
if self.startup_fill:
self._fill_all_kv_caches()
def _fill_all_kv_caches(self):
"""Fill the KV cache tensors of the startup-filled groups once, in
place, with dummy values, so that no per-request fill is needed.
"""
assert self.kv_caches is not None
num_layers = 0
for group_idx in sorted(self._startup_fill_group_ids):
for layer_name in self.group_to_layers[group_idx]:
kv_cache = self.kv_caches.get(layer_name)
if isinstance(kv_cache, torch.Tensor):
tensors = [kv_cache]
elif isinstance(kv_cache, (list, tuple)) and all(
isinstance(t, torch.Tensor) for t in kv_cache
):
tensors = list(kv_cache)
else:
logger.warning_once(
"DecodeBenchConnector: skipping startup fill for layer "
"%s whose KV cache is %s, not a tensor or a list/tuple "
"of tensors.",
layer_name,
type(kv_cache).__name__,
)
continue
for tensor in tensors:
fill_dtype = self._fp8_dtypes.get(layer_name, tensor.dtype)
self._fill_tensor(tensor, self.fill_mean, self.fill_std, fill_dtype)
num_layers += 1
logger.info(
"DecodeBenchConnector: Filled %d KV cache layers at startup with %s "
"values (mean=%.3f, std=%.3f)",
num_layers,
"random" if self.fill_std > 0 else "constant",
self.fill_mean,
self.fill_std,
)
def start_fill_kv(self, metadata: DecodeBenchConnectorMetadata):
"""Fill the allocated KV cache blocks with dummy values.
This simulates having a populated KV cache from a prefill phase,
allowing decode performance testing with larger context sizes.
Supports both single- and multi-group KV cache configurations.
"""
if not metadata.reqs_to_fill:
return
assert self.kv_caches is not None, "KV caches must be registered before filling"
for req_id, (block_ids_per_group, num_tokens) in metadata.reqs_to_fill.items():
# Fill blocks for each KV cache group
for group_idx, block_ids in enumerate(block_ids_per_group):
self._fill_blocks(group_idx, block_ids, num_tokens)
block_counts = tuple(len(group) for group in block_ids_per_group)
logger.debug(
"DecodeBenchConnector: Filled %d total blocks (%d tokens) across "
"%d groups for request %s (per-group counts: %s)",
sum(block_counts),
num_tokens,
len(block_ids_per_group),
req_id,
", ".join(map(str, block_counts)),
)
def _fill_blocks(self, group_idx: int, block_ids: list[int], num_tokens: int):
"""Fill specified blocks with dummy values for a specific KV cache group.
Args:
group_idx: The KV cache group index to fill
block_ids: List of block IDs to fill in this group
num_tokens: Total number of tokens to fill across these blocks
"""
if not block_ids:
return
assert self.kv_caches is not None
# Circular buffers may pack non-floating metadata alongside their
# floating-point state, so arbitrary fill values are not representation-safe.
fill_mean, fill_std = (
(0.0, 0.0)
if group_idx in self._zero_fill_group_ids
else (self.fill_mean, self.fill_std)
)
# Get the layers that belong to this group
layer_names = self.group_to_layers.get(group_idx, [])
# Fill only the layers in this group
for layer_name in layer_names:
if layer_name not in self.kv_caches:
logger.warning(
"DecodeBenchConnector: Layer %s not found in KV caches", layer_name
)
continue
kv_cache = self.kv_caches[layer_name]
# Attention layers store KV as a single block-indexed tensor whose
# first dim is num_blocks; fill the requested block rows. Hybrid /
# linear-attention layers (e.g. Mamba, Kimi Delta Attention) store
# their state as a list/tuple of tensors that are NOT block-indexed
# — each tensor is a single state buffer with no num_blocks
# dimension — so fill each tensor in its entirety with the same
# dummy values.
if isinstance(kv_cache, torch.Tensor):
fill_dtype = self._fp8_dtypes.get(layer_name, kv_cache.dtype)
self._fill_block_tensor(
kv_cache, block_ids, fill_mean, fill_std, fill_dtype
)
elif isinstance(kv_cache, (list, tuple)) and all(
isinstance(t, torch.Tensor) for t in kv_cache
):
for state_tensor in kv_cache:
self._fill_tensor(
state_tensor, fill_mean, fill_std, state_tensor.dtype
)
else:
logger.warning_once(
"DecodeBenchConnector: skipping fill for layer %s whose KV "
"cache is %s, not a tensor or a list/tuple of tensors.",
layer_name,
type(kv_cache).__name__,
)
continue
logger.debug(
"DecodeBenchConnector: Filled %d blocks in group %d with %s values "
"(mean=%.3f, std=%.3f)",
len(block_ids),
group_idx,
"random" if fill_std > 0 else "constant",
fill_mean,
fill_std,
)
def _fill_block_tensor(
self,
kv_cache: torch.Tensor,
block_ids: list[int],
fill_mean: float,
fill_std: float,
fill_dtype: torch.dtype,
):
"""Fill the requested block rows of a block-indexed KV cache tensor.
Args:
kv_cache: A KV cache tensor whose first dim is num_blocks.
block_ids: Block IDs to fill. IDs that are out of range for this
tensor's first dim are ignored.
fill_mean: Mean value for the fill.
fill_std: Standard deviation for the fill.
fill_dtype: Dtype the values are encoded as, e.g. the fp8 dtype
of a uint8 fp8 cache.
"""
# Convert block_ids to tensor on device
block_ids_tensor = torch.tensor(
block_ids, dtype=torch.long, device=kv_cache.device
)
# Filter invalid block IDs
valid_mask = block_ids_tensor < kv_cache.shape[0]
valid_block_ids = block_ids_tensor[valid_mask]
if len(valid_block_ids) == 0:
return
fill_values = self._make_fill_values(
(len(valid_block_ids),) + kv_cache.shape[1:],
kv_cache.device,
fill_dtype,
fill_mean,
fill_std,
)
# Batch fill operation
kv_cache[valid_block_ids] = fill_values.view(kv_cache.dtype)
def _make_fill_values(
self,
size: tuple[int, ...],
device: torch.device,
dtype: torch.dtype,
fill_mean: float,
fill_std: float,
) -> torch.Tensor:
"""Create constant or random fill values of ``dtype``, clamped to its
finite range. Non-floating dtypes (packed layouts) are filled with
zeros.
"""
if not dtype.is_floating_point:
if fill_mean or fill_std:
logger.warning_once(
"DecodeBenchConnector: %s KV caches do not hold plain "
"floating-point values; filling them with zeros.",
dtype,
)
return torch.zeros(size, dtype=dtype, device=device)
finfo = torch.finfo(dtype)
if fill_std > 0:
sample_dtype = torch.float32
if dtype in (torch.float16, torch.bfloat16, torch.float32, torch.float64):
sample_dtype = dtype
values = torch.normal(
fill_mean, fill_std, size=size, device=device, dtype=sample_dtype
)
return values.clamp_(finfo.min, finfo.max).to(dtype)
fill_mean = min(max(fill_mean, finfo.min), finfo.max)
return torch.full(size, fill_mean, dtype=dtype, device=device)
def _fill_tensor(
self,
kv_cache: torch.Tensor,
fill_mean: float,
fill_std: float,
fill_dtype: torch.dtype,
):
"""Fill an entire tensor in place with dummy values.
Used for startup fills, and for hybrid / linear-attention layers (e.g.
Mamba, Kimi Delta Attention) whose per-layer state tensors are filled
in their entirety with the same constant or random values used for
block fills, rather than selected block rows.
Args:
kv_cache: A tensor to fill in its entirety.
fill_mean: Mean value for the fill.
fill_std: Standard deviation for the fill.
fill_dtype: Dtype the values are encoded as, e.g. the fp8 dtype
of a uint8 fp8 cache.
"""
view = kv_cache.view(fill_dtype)
if fill_std == 0 or not fill_dtype.is_floating_point:
view.fill_(
self._make_fill_values(
(), torch.device("cpu"), fill_dtype, fill_mean, fill_std
)
)
return
# Sample in chunks to bound the temporary memory.
rows = max(1, (1 << 24) // max(1, view[0].numel()))
for chunk in view.split(rows):
chunk.copy_(
self._make_fill_values(
chunk.shape, chunk.device, fill_dtype, fill_mean, fill_std
)
)
vllm/distributed/kv_transfer/kv_connector/v1/offloading/events.py
-18
View File
@@ -87,8 +87,6 @@ class _OffloadEventMetadata:
parent_block_hash: BlockHash | None
token_ids: tuple[int, ...]
block_size: int
lora_id: int | None
lora_name: str | None
# Deferred: needs the same incremental curr_mm_idx handling as GPU events.
extra_keys: tuple[tuple[Any, ...] | None, ...] | None
group_idx: int
@@ -218,15 +216,11 @@ class OffloadingEventsTracker:
)
assert first_hash_idx == 0 or parent_block_hash is not None
lora_id = req.lora_request.adapter_id if req.lora_request is not None else None
lora_name = req.lora_request.name if req.lora_request is not None else None
meta = _OffloadEventMetadata(
block_hashes=block_hashes,
parent_block_hash=parent_block_hash,
token_ids=tuple(req.all_token_ids[chunk_start:boundary_tokens]),
block_size=tokens_per_hash,
lora_id=lora_id,
lora_name=lora_name,
extra_keys=None,
group_idx=group_config.group_idx,
kv_cache_spec=group_config.kv_event_group_spec,
@@ -315,19 +309,11 @@ class OffloadingEventsTracker:
assert tok_end <= len(req.all_token_ids)
token_ids = tuple(req.all_token_ids[tok_start:tok_end])
lora_id: int | None = None
lora_name: str | None = None
if req.lora_request is not None:
lora_id = req.lora_request.adapter_id
lora_name = req.lora_request.name
return _OffloadEventMetadata(
block_hashes=tuple(chunk_hashes),
parent_block_hash=parent_block_hash,
token_ids=token_ids,
block_size=group_config.tokens_per_block,
lora_id=lora_id,
lora_name=lora_name,
extra_keys=None,
group_idx=group_config.group_idx,
kv_cache_spec=group_config.kv_event_group_spec,
@@ -347,10 +333,8 @@ class OffloadingEventsTracker:
],
parent_block_hash=None,
token_ids=[],
lora_id=None,
block_size=0,
medium=_MEDIUM_TO_EVENT_STR[medium],
lora_name=None,
group_idx=get_offload_group_idx(key),
locality=locality,
ownership=ownership,
@@ -391,9 +375,7 @@ class OffloadingEventsTracker:
),
token_ids=list(meta.token_ids),
block_size=meta.block_size,
lora_id=meta.lora_id,
medium=_MEDIUM_TO_EVENT_STR[event.medium],
lora_name=meta.lora_name,
extra_keys=to_event_extra_keys(meta.extra_keys),
group_idx=meta.group_idx,
kv_cache_spec_kind=meta.kv_cache_spec.kv_cache_spec_kind,
vllm/engine/arg_utils.py
+1 -119
View File
@@ -28,7 +28,6 @@ from typing import (
import huggingface_hub
import regex as re
import torch
from pydantic import TypeAdapter, ValidationError
from pydantic.fields import FieldInfo
from typing_extensions import TypeIs
@@ -52,7 +51,6 @@ from vllm.config import (
KVTransferConfig,
LoadConfig,
LoggingConfig,
LoRAConfig,
MambaConfig,
ModelConfig,
MultiModalConfig,
@@ -88,7 +86,6 @@ from vllm.config.kernel import (
)
from vllm.config.load import SafetensorsLoadStrategy
from vllm.config.logging import LogLevel
from vllm.config.lora import MaxLoRARanks
from vllm.config.mamba import MambaBackendEnum, MambaSSUAlgorithm
from vllm.config.model import (
ConvertOption,
@@ -119,7 +116,6 @@ from vllm.config.vllm import OptimizationLevel, PerformanceMode
from vllm.config.watermarking import WatermarkConfig
from vllm.logger import configure_logging_if_needed, init_logger, suppress_logging
from vllm.platforms import CpuArchEnum, current_platform
from vllm.plugins import load_general_plugins
from vllm.ray.lazy_utils import is_in_ray_actor, is_ray_initialized
from vllm.transformers_utils.config import maybe_override_with_speculators
from vllm.transformers_utils.repo_utils import get_model_path
@@ -682,20 +678,6 @@ class EngineArgs:
mm_processor_device: MMProcessorDevice = "auto"
mm_ipc_gpu_memory_gb: float = MultiModalConfig.mm_ipc_gpu_memory_gb
mm_device_do_normalize: bool | None = MultiModalConfig.mm_device_do_normalize
# LoRA fields
enable_lora: bool = False
max_loras: int = LoRAConfig.max_loras
max_lora_rank: MaxLoRARanks = LoRAConfig.max_lora_rank
default_mm_loras: dict[str, str] | None = LoRAConfig.default_mm_loras
fully_sharded_loras: bool = LoRAConfig.fully_sharded_loras
max_cpu_loras: int | None = LoRAConfig.max_cpu_loras
lora_dtype: str | torch.dtype | None = LoRAConfig.lora_dtype
lora_target_modules: list[str] | None = LoRAConfig.target_modules
enable_tower_connector_lora: bool = LoRAConfig.enable_tower_connector_lora
specialize_active_lora: bool = LoRAConfig.specialize_active_lora
enable_mixed_moe_lora_format: bool = LoRAConfig.enable_mixed_moe_lora_format
enable_moe_shared_loras: bool = LoRAConfig.enable_moe_shared_loras
max_lora_cls_labels: int | None = LoRAConfig.max_lora_cls_labels
ray_workers_use_nsight: bool = ParallelConfig.ray_workers_use_nsight
num_gpu_blocks_override: int | None = CacheConfig.num_gpu_blocks_override
@@ -848,7 +830,6 @@ class EngineArgs:
kv_offloading_size: float | None = CacheConfig.kv_offloading_size
kv_offloading_backend: KVOffloadingBackend = CacheConfig.kv_offloading_backend
tokens_only: bool = False
shutdown_timeout: int = 0
@@ -911,10 +892,6 @@ class EngineArgs:
self.quantization, self.quantization_config
)
# Setup plugins
from vllm.plugins import load_general_plugins
load_general_plugins()
# when use hf offline,replace model and tokenizer id to local model path
if huggingface_hub.constants.HF_HUB_OFFLINE:
# Skip cloud storage URIs (s3://, gs://, az://) — they are not
@@ -1579,51 +1556,6 @@ class EngineArgs:
},
)
# LoRA related configs
lora_kwargs = get_kwargs(LoRAConfig)
lora_group = parser.add_argument_group(
title="LoRAConfig",
description=LoRAConfig.__doc__,
)
lora_group.add_argument(
"--enable-lora",
action=argparse.BooleanOptionalAction,
help="If True, enable handling of LoRA adapters.",
)
lora_group.add_argument("--max-loras", **lora_kwargs["max_loras"])
lora_group.add_argument("--max-lora-rank", **lora_kwargs["max_lora_rank"])
lora_group.add_argument(
"--lora-dtype",
**lora_kwargs["lora_dtype"],
)
lora_group.add_argument(
"--enable-tower-connector-lora",
**lora_kwargs["enable_tower_connector_lora"],
)
lora_group.add_argument("--max-cpu-loras", **lora_kwargs["max_cpu_loras"])
lora_group.add_argument(
"--fully-sharded-loras", **lora_kwargs["fully_sharded_loras"]
)
lora_group.add_argument(
"--lora-target-modules", **lora_kwargs["target_modules"]
)
lora_group.add_argument("--default-mm-loras", **lora_kwargs["default_mm_loras"])
lora_group.add_argument(
"--specialize-active-lora", **lora_kwargs["specialize_active_lora"]
)
lora_group.add_argument(
"--enable-mixed-moe-lora-format",
**lora_kwargs["enable_mixed_moe_lora_format"],
)
lora_group.add_argument(
"--enable-moe-shared-loras",
**lora_kwargs["enable_moe_shared_loras"],
)
lora_group.add_argument(
"--max-lora-cls-labels",
**lora_kwargs["max_lora_cls_labels"],
)
# Logging arguments
logging_group = parser.add_argument_group(
title="LoggingConfig",
@@ -2537,10 +2469,6 @@ class EngineArgs:
else ParallelConfig.data_parallel_rpc_port
)
if self.tokens_only and not model_config.skip_tokenizer_init:
model_config.skip_tokenizer_init = True
logger.info("Skipping tokenizer initialization for tokens-only mode.")
parallel_config = ParallelConfig(
pipeline_parallel_size=self.pipeline_parallel_size,
tensor_parallel_size=self.tensor_parallel_size,
@@ -2657,47 +2585,6 @@ class EngineArgs:
stream_interval=self.stream_interval,
)
if not model_config.is_multimodal_model and self.default_mm_loras:
raise ValueError(
"Default modality-specific LoRA(s) were provided for a "
"non multimodal model"
)
lora_config = (
LoRAConfig(
max_lora_rank=self.max_lora_rank,
max_loras=self.max_loras,
default_mm_loras=self.default_mm_loras,
fully_sharded_loras=self.fully_sharded_loras,
lora_dtype=self.lora_dtype,
target_modules=self.lora_target_modules,
enable_tower_connector_lora=self.enable_tower_connector_lora,
specialize_active_lora=self.specialize_active_lora,
enable_mixed_moe_lora_format=self.enable_mixed_moe_lora_format,
enable_moe_shared_loras=self.enable_moe_shared_loras,
max_lora_cls_labels=self.max_lora_cls_labels,
max_cpu_loras=self.max_cpu_loras
if self.max_cpu_loras and self.max_cpu_loras > 0
else None,
)
if self.enable_lora
else None
)
if (
lora_config is not None
and speculative_config is not None
and scheduler_config.max_num_batched_tokens
< (
scheduler_config.max_num_seqs
* (speculative_config.num_speculative_tokens + 1)
)
):
raise ValueError(
"Consider increasing max_num_batched_tokens or "
"decreasing num_speculative_tokens"
)
# Attention config overrides
attention_config = copy.deepcopy(self.attention_config)
if self.attention_backend is not None:
@@ -2867,7 +2754,6 @@ class EngineArgs:
engram_config=self.engram_config,
mamba_config=mamba_config,
kernel_config=kernel_config,
lora_config=lora_config,
speculative_config=speculative_config,
watermark_config=watermark_config,
diffusion_config=diffusion_config,
@@ -3223,10 +3109,6 @@ class AsyncEngineArgs(EngineArgs):
def add_cli_args(
parser: FlexibleArgumentParser, async_args_only: bool = False
) -> FlexibleArgumentParser:
# Initialize plugin to update the parser, for example, The plugin may
# add a new kind of quantization method to --quantization argument or
# a new device to --device argument.
load_general_plugins()
if not async_args_only:
parser = EngineArgs.add_cli_args(parser)
parser.add_argument(
@@ -3234,7 +3116,7 @@ class AsyncEngineArgs(EngineArgs):
action=argparse.BooleanOptionalAction,
default=AsyncEngineArgs.enable_log_requests,
help="Enable logging request information, dependent on log level:\n"
"- INFO: Request ID, parameters and LoRA request.\n"
"- INFO: Request ID and parameters.\n"
"- DEBUG: Prompt inputs (e.g: text, token IDs).\n"
"You can set the minimum log level via `VLLM_LOGGING_LEVEL`.\n"
"You can also set a separate log level for request JSON body "
vllm/engine/protocol.py
-8
View File
@@ -13,7 +13,6 @@ from vllm.distributed.weight_transfer.base import (
WeightTransferUpdateRequest,
)
from vllm.inputs import EngineInput, PromptType
from vllm.lora.request import LoRARequest
from vllm.outputs import PoolingRequestOutput, RequestOutput
from vllm.pooling_params import PoolingParams
from vllm.renderers import BaseRenderer
@@ -101,7 +100,6 @@ class EngineClient(ABC):
request_id: str,
*,
prompt_text: str | None = None,
lora_request: LoRARequest | None = None,
tokenization_kwargs: dict[str, Any] | None = None,
trace_headers: Mapping[str, str] | None = None,
priority: int = 0,
@@ -120,7 +118,6 @@ class EngineClient(ABC):
prompt: PromptType | EngineInput,
pooling_params: PoolingParams,
request_id: str,
lora_request: LoRARequest | None = None,
trace_headers: Mapping[str, str] | None = None,
priority: int = 0,
tokenization_kwargs: dict[str, Any] | None = None,
@@ -218,11 +215,6 @@ class EngineClient(ABC):
"""Check whether the engine is sleeping."""
...
@abstractmethod
async def add_lora(self, lora_request: LoRARequest) -> bool:
"""Load a new LoRA adapter into the engine for future requests."""
...
@abstractmethod
async def pause_generation(
self,
vllm/entrypoints/cli/serve.py
-6
View File
@@ -8,7 +8,6 @@ import time
import uvloop
import vllm
import vllm.envs as envs
from vllm.entrypoints.cli.types import CLISubcommand
from vllm.entrypoints.launchers.api_server.entry import run_server, setup_server
from vllm.entrypoints.launchers.cli_args import (
@@ -284,11 +283,6 @@ def run_multi_api_server(args: argparse.Namespace):
usage_context = UsageContext.OPENAI_API_SERVER
vllm_config = engine_args.create_engine_config(usage_context=usage_context)
if num_api_servers > 1 and envs.VLLM_ALLOW_RUNTIME_LORA_UPDATING:
raise ValueError(
"VLLM_ALLOW_RUNTIME_LORA_UPDATING cannot be used with api_server_count > 1"
)
executor_class = Executor.get_class(vllm_config)
log_stats = not engine_args.disable_log_stats
vllm/entrypoints/common/offline.py
-88
View File
@@ -18,7 +18,6 @@ from vllm.config import ModelConfig
from vllm.exceptions import VLLMValidationError
from vllm.inputs import EngineInput
from vllm.logger import init_logger
from vllm.lora.request import LoRARequest
from vllm.renderers import BaseRenderer, ChatParams, merge_kwargs
from vllm.renderers.chat_utils import (
ChatCompletionMessageParam,
@@ -55,58 +54,6 @@ class OfflineInferenceMixin:
model_config: ModelConfig
chat_template: str | None
def _resolve_mm_lora(
self,
prompt: EngineInput,
lora_request: LoRARequest | None,
) -> LoRARequest | None:
if prompt["type"] != "multimodal":
return lora_request
lora_config = self.llm_engine.vllm_config.lora_config
default_mm_loras = None if lora_config is None else lora_config.default_mm_loras
if not default_mm_loras:
return lora_request
prompt_modalities = prompt["mm_placeholders"].keys()
intersection = set(prompt_modalities).intersection(default_mm_loras.keys())
if not intersection:
return lora_request
if len(intersection) > 1:
# TODO: Would be nice to be able to have multiple loras per prompt
logger.warning(
"Multiple modality specific loras were registered and would be "
"used by a single prompt consuming several modalities; "
"currently we only support one lora per request; as such, "
"lora(s) registered with modalities: %s will be skipped",
intersection,
)
return lora_request
# Build the LoRA request; the ID of the default mm lora is the
# index of the modality name sorted alphabetically + 1.
modality_name = intersection.pop()
modality_lora_path = default_mm_loras[modality_name]
modality_lora_id = sorted(default_mm_loras).index(modality_name) + 1
# If we have a collision, warn if there is a collision,
# but always send the explicitly provided request.
if lora_request:
if lora_request.lora_int_id != modality_lora_id:
logger.warning(
"A modality with a registered lora and a lora_request "
"with a different ID were provided; falling back to the "
"lora_request as we only apply one LoRARequest per prompt"
)
return lora_request
return LoRARequest(
modality_name,
modality_lora_id,
modality_lora_path,
)
def _preprocess_cmpl(
self,
prompts: Sequence[PromptType],
@@ -262,22 +209,6 @@ class OfflineInferenceMixin:
return [params] * num_requests
def _lora_request_to_seq(
self,
lora_request: LoRARequest | None | Sequence[LoRARequest | None],
num_requests: int,
) -> Sequence[LoRARequest | None]:
if isinstance(lora_request, Sequence):
if len(lora_request) != num_requests:
raise VLLMValidationError(
f"The lengths of prompts ({num_requests}) "
f"and lora_request ({len(lora_request)}) must be the same."
)
return lora_request
return [lora_request] * num_requests
def _priority_to_seq(
self,
priority: int | Sequence[int] | None,
@@ -305,14 +236,12 @@ class OfflineInferenceMixin:
| Sequence[SamplingParams | PoolingParams],
*,
use_tqdm: bool | Callable[..., tqdm] = True,
lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
priority: int | Sequence[int] | None = None,
tokenization_kwargs: dict[str, Any] | None = None,
mm_processor_kwargs: dict[str, Any] | None = None,
) -> list[str]:
seq_prompts = prompt_to_seq(prompts)
seq_params = self._params_to_seq(params, len(seq_prompts))
seq_lora_requests = self._lora_request_to_seq(lora_request, len(seq_prompts))
seq_priority = self._priority_to_seq(priority, len(seq_prompts))
return self._render_and_add_requests(
@@ -329,7 +258,6 @@ class OfflineInferenceMixin:
)
),
params=seq_params,
lora_requests=seq_lora_requests,
priorities=seq_priority,
)
@@ -342,7 +270,6 @@ class OfflineInferenceMixin:
output_type: type[_O],
*,
use_tqdm: bool | Callable[..., tqdm] = True,
lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
priority: int | Sequence[int] | None = None,
tokenization_kwargs: dict[str, Any] | None = None,
mm_processor_kwargs: dict[str, Any] | None = None,
@@ -351,7 +278,6 @@ class OfflineInferenceMixin:
prompts=prompts,
params=params,
use_tqdm=use_tqdm,
lora_request=lora_request,
priority=priority,
tokenization_kwargs=tokenization_kwargs,
mm_processor_kwargs=mm_processor_kwargs,
@@ -368,7 +294,6 @@ class OfflineInferenceMixin:
output_type: type[_O],
*,
use_tqdm: bool | Callable[..., tqdm] = True,
lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
priority: int | Sequence[int] | None = None,
chat_template: str | None = None,
chat_template_content_format: ChatTemplateContentFormatOption = "auto",
@@ -383,7 +308,6 @@ class OfflineInferenceMixin:
messages=messages,
params=params,
use_tqdm=use_tqdm,
lora_request=lora_request,
priority=priority,
chat_template=chat_template,
chat_template_content_format=chat_template_content_format,
@@ -405,7 +329,6 @@ class OfflineInferenceMixin:
| Sequence[SamplingParams | PoolingParams],
*,
use_tqdm: bool | Callable[..., tqdm] = True,
lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
priority: int | Sequence[int] | None = None,
chat_template: str | None = None,
chat_template_content_format: ChatTemplateContentFormatOption = "auto",
@@ -418,7 +341,6 @@ class OfflineInferenceMixin:
) -> list[str]:
seq_convs = conversation_to_seq(messages)
seq_params = self._params_to_seq(params, len(seq_convs))
seq_lora_requests = self._lora_request_to_seq(lora_request, len(seq_convs))
seq_priority = self._priority_to_seq(priority, len(seq_convs))
# When thinking is enabled or tools are provided, and the model
@@ -452,7 +374,6 @@ class OfflineInferenceMixin:
)
),
params=seq_params,
lora_requests=seq_lora_requests,
priorities=seq_priority,
)
@@ -509,7 +430,6 @@ class OfflineInferenceMixin:
params: Sequence[SamplingParams | PoolingParams],
output_type: type[_O],
*,
lora_requests: Sequence[LoRARequest | None] | None = None,
priorities: Sequence[int] | None = None,
use_tqdm: bool | Callable[..., tqdm] = True,
):
@@ -526,7 +446,6 @@ class OfflineInferenceMixin:
self._render_and_add_requests(
prompts=prompts,
params=params,
lora_requests=lora_requests,
priorities=priorities,
)
@@ -537,7 +456,6 @@ class OfflineInferenceMixin:
prompts: Iterable[EngineInput],
params: Sequence[SamplingParams | PoolingParams],
*,
lora_requests: Sequence[LoRARequest | None] | None = None,
priorities: Sequence[int] | None = None,
) -> list[str]:
added_request_ids: list[str] = []
@@ -547,10 +465,6 @@ class OfflineInferenceMixin:
request_id = self._add_request(
prompt,
params[i],
lora_request=self._resolve_mm_lora(
prompt,
None if lora_requests is None else lora_requests[i],
),
priority=0 if priorities is None else priorities[i],
)
added_request_ids.append(request_id)
@@ -565,7 +479,6 @@ class OfflineInferenceMixin:
self,
prompt: EngineInput,
params: SamplingParams | PoolingParams,
lora_request: LoRARequest | None = None,
priority: int = 0,
) -> str:
if isinstance(params, SamplingParams):
@@ -578,7 +491,6 @@ class OfflineInferenceMixin:
request_id,
prompt,
params,
lora_request=lora_request,
priority=priority,
)
vllm/entrypoints/generate/base/serving.py
-2
View File
@@ -29,7 +29,6 @@ from vllm.exceptions import GenerationError
from vllm.inputs import EngineInput
from vllm.logger import init_logger
from vllm.logprobs import Logprob, PromptLogprobs
from vllm.lora.request import LoRARequest
from vllm.tokenizers import TokenizerLike
from vllm.tokenizers.detokenizer_utils import convert_ids_list_to_tokens
from vllm.v1.metrics.stats import RequestStateStats
@@ -121,7 +120,6 @@ class ServeContext(Generic[RequestT]):
model_name: str
request_id: str
created_time: int = field(default_factory=lambda: int(time.time()))
lora_request: LoRARequest | None = None
engine_inputs: list[EngineInput] | None = None
model_config = ConfigDict(arbitrary_types_allowed=True)
vllm/entrypoints/generate/beam_search/offline.py
+1 -8
View File
@@ -11,7 +11,6 @@ from tqdm import tqdm
from vllm import RequestOutput, TextPrompt, TokensPrompt
from vllm.entrypoints.common.offline import OfflineInferenceMixin
from vllm.logger import init_logger
from vllm.lora.request import LoRARequest
from vllm.pooling_params import PoolingParams
from vllm.sampling_params import (
BeamSearchParams,
@@ -60,7 +59,6 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin):
self,
prompts: list[TokensPrompt | TextPrompt],
params: BeamSearchParams,
lora_request: list[LoRARequest] | LoRARequest | None = None,
use_tqdm: bool = False,
concurrency_limit: int | None = None,
) -> list[BeamSearchOutput]:
@@ -70,7 +68,6 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin):
prompts: A list of prompts. Each prompt can be a string or a list
of token IDs.
params: The beam search parameters.
lora_request: LoRA request to use for generation, if any.
use_tqdm: Whether to use tqdm to display the progress bar.
concurrency_limit: The maximum number of concurrent requests.
If None, the number of concurrent requests is unlimited.
@@ -90,7 +87,6 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin):
sort_beams_key = create_sort_beams_key_function(eos_token_id, length_penalty)
engine_inputs = self._preprocess_cmpl(prompts)
lora_requests = self._lora_request_to_seq(lora_request, len(engine_inputs))
if use_tqdm and concurrency_limit is not None:
logger.warning(
@@ -131,7 +127,7 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin):
instances: list[BeamSearchInstance] = []
for lora_req, prompt in zip(lora_requests, engine_inputs):
for prompt in engine_inputs:
if prompt["type"] == "embeds":
raise NotImplementedError(
"Embedding prompt not supported for beam search"
@@ -140,7 +136,6 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin):
instances.append(
BeamSearchInstance(
prompt,
lora_request=lora_req,
logprobs=None,
),
)
@@ -279,7 +274,6 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin):
prompts=(beam.get_prompt() for beam in active_beams),
params=active_params,
output_type=RequestOutput,
lora_requests=[beam.lora_request for beam in active_beams],
use_tqdm=False,
)
@@ -346,7 +340,6 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin):
current_beam.orig_prompt,
tokens=current_beam.tokens + [token_id],
logprobs=current_beam.logprobs + [logprobs],
lora_request=current_beam.lora_request,
cum_logprob=current_beam.cum_logprob + logprob_obj.logprob,
)
vllm/entrypoints/generate/beam_search/online.py
-6
View File
@@ -11,7 +11,6 @@ import numpy as np
from vllm import CompletionOutput, RequestOutput
from vllm.engine.protocol import EngineClient
from vllm.inputs import EngineInput
from vllm.lora.request import LoRARequest
from vllm.renderers import BaseRenderer
from vllm.sampling_params import BeamSearchParams, SamplingParams
from vllm.utils import random_uuid
@@ -34,7 +33,6 @@ class BeamSearchOnlineMixin(ABC):
prompt: EngineInput,
request_id: str,
params: BeamSearchParams,
lora_request: LoRARequest | None = None,
trace_headers: Mapping[str, str] | None = None,
session_id: str | None = None,
) -> AsyncGenerator[RequestOutput, None]:
@@ -80,7 +78,6 @@ class BeamSearchOnlineMixin(ABC):
tokens=prompt_token_ids,
cum_logprob=0,
logprobs=[],
lora_request=lora_request,
)
]
completed = []
@@ -91,7 +88,6 @@ class BeamSearchOnlineMixin(ABC):
for i, beam in enumerate(all_beams):
prompt_item = beam.get_prompt()
lora_request_item = beam.lora_request
request_id_item = f"{request_id_batch}-beam-{i}"
task = asyncio.create_task(
collect_from_async_generator(
@@ -99,7 +95,6 @@ class BeamSearchOnlineMixin(ABC):
prompt_item,
sampling_params,
request_id_item,
lora_request=lora_request_item,
trace_headers=trace_headers,
session_id=session_id,
)
@@ -205,7 +200,6 @@ class BeamSearchOnlineMixin(ABC):
orig_prompt=prompt,
tokens=current_beam.tokens + [token_id],
logprobs=current_beam.logprobs + [logprobs],
lora_request=current_beam.lora_request,
cum_logprob=cum_logprob,
)
)
vllm/entrypoints/generate/beam_search/utils.py
-4
View File
@@ -12,7 +12,6 @@ from vllm.inputs import (
tokens_input,
)
from vllm.logprobs import Logprob
from vllm.lora.request import LoRARequest
@dataclass
@@ -28,7 +27,6 @@ class BeamSearchSequence:
# NOTE: Tokens represents decoder tokens in the encoder / decoder case
tokens: list[int]
logprobs: list[dict[int, Logprob]]
lora_request: LoRARequest | None = None
cum_logprob: float = 0.0
text: str | None = None
finish_reason: str | None = None
@@ -113,7 +111,6 @@ class BeamSearchInstance:
def __init__(
self,
prompt: TokensInput | MultiModalInput | EncoderDecoderInput,
lora_request: LoRARequest | None = None,
logprobs: list[dict[int, Logprob]] | None = None,
**kwargs,
):
@@ -127,7 +124,6 @@ class BeamSearchInstance:
orig_prompt=prompt,
tokens=initial_tokens,
logprobs=[] if logprobs is None else list(logprobs),
lora_request=lora_request,
**kwargs,
)
]
vllm/entrypoints/generate/generative_scoring/serving.py
+1 -9
View File
@@ -245,12 +245,6 @@ class ServingGenerativeScoring(BaseServing):
# Note: Mixed item types (string and token list) are validated by
# Pydantic at request parsing time, so we don't need to check here.
try:
lora_request = self._maybe_get_adapters(request) # type: ignore[arg-type]
except (ValueError, TypeError, RuntimeError) as e:
logger.exception("Error preparing request components")
return self.create_error_response(e)
base_id = self._base_request_id(raw_request, default=request.request_id)
request_id = f"generative-scoring-{base_id}"
created_time = int(time.time())
@@ -290,7 +284,6 @@ class ServingGenerativeScoring(BaseServing):
f"{request_id}-{i}",
engine_input,
params=sampling_params,
lora_request=lora_request,
)
try:
@@ -299,7 +292,6 @@ class ServingGenerativeScoring(BaseServing):
engine_inputs,
[sampling_params] * len(engine_inputs),
request_id,
lora_request=lora_request,
trace_headers=trace_headers,
priority=request.priority,
)
@@ -343,7 +335,7 @@ class ServingGenerativeScoring(BaseServing):
total_completion_tokens += len(read.result.outputs[0].token_ids)
# Build response
model_name = self.models.model_name(lora_request)
model_name = self.models.model_name()
response = GenerativeScoringResponse(
id=request_id,
created=created_time,
vllm/entrypoints/generate/label_reads.py
-3
View File
@@ -8,7 +8,6 @@ from dataclasses import dataclass
from vllm.engine.protocol import EngineClient
from vllm.exceptions import GenerationError
from vllm.inputs import EngineInput
from vllm.lora.request import LoRARequest
from vllm.outputs import RequestOutput
from vllm.sampling_params import SamplingParams
from vllm.utils.async_utils import merge_async_iterators
@@ -28,7 +27,6 @@ async def next_token_label_reads(
sampling_params: Sequence[SamplingParams],
request_id: str,
*,
lora_request: LoRARequest | None = None,
trace_headers: Mapping[str, str] | None = None,
priority: int = 0,
) -> list[LabelRead]:
@@ -41,7 +39,6 @@ async def next_token_label_reads(
engine_input,
params,
f"{request_id}-{i}",
lora_request=lora_request,
trace_headers=trace_headers,
priority=priority,
)
vllm/entrypoints/launchers/api_server/app_state.py
-24
View File
@@ -11,9 +11,7 @@ from vllm.entrypoints.mcp.tool_server import init_tool_server
from vllm.entrypoints.openai.models.protocol import BaseModelPath
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
from vllm.entrypoints.serve.utils.api_utils import process_lora_modules
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.plugins.endpoint_plugins.interface import init_endpoint_plugins_state
from vllm.renderers.chat_utils import load_chat_template
from vllm.renderers.online_renderer import OnlineRenderer
from vllm.tasks import FALLBACK_SUPPORTED_TASKS, SupportedTask
@@ -68,20 +66,10 @@ async def init_app_state(
await init_tool_server(args) if "generate" in supported_tasks else None
)
# Merge default_mm_loras into the static lora_modules
default_mm_loras = (
vllm_config.lora_config.default_mm_loras
if vllm_config.lora_config is not None
else {}
)
lora_modules = process_lora_modules(args.lora_modules, default_mm_loras)
state.openai_serving_models = OpenAIServingModels(
engine_client=engine_client,
base_model_paths=base_model_paths,
lora_modules=lora_modules,
)
await state.openai_serving_models.init_static_loras()
state.online_renderer = OnlineRenderer(
model_config=engine_client.model_config,
@@ -123,18 +111,6 @@ async def init_app_state(
default_chat_template_kwargs,
)
from vllm.entrypoints.scale_out.factories import init_scale_out_state
init_scale_out_state(state, args, engine_client, request_logger)
if "transcription" in supported_tasks or "realtime" in supported_tasks:
from vllm.entrypoints.speech_to_text.factories import init_speech_to_text_state
init_speech_to_text_state(
engine_client, state, args, request_logger, supported_tasks
)
await init_endpoint_plugins_state(engine_client, state, args)
state.enable_server_load_tracking = args.enable_server_load_tracking
state.server_load_metrics = 0
vllm/entrypoints/launchers/api_server/routers.py
-11
View File
@@ -31,17 +31,6 @@ def register_api_routers(
register_generate_api_routers(app)
from vllm.entrypoints.scale_out.factories import register_scale_out_api_routers
register_scale_out_api_routers(app, supported_tasks)
if "transcription" in supported_tasks or "realtime" in supported_tasks:
from vllm.entrypoints.speech_to_text.factories import (
register_speech_to_text_api_routers,
)
register_speech_to_text_api_routers(app, supported_tasks)
if getattr(args, "enable_fault_tolerance", False):
from vllm.entrypoints.serve.fault_tolerance.api_router import (
register_fault_tolerance_api_router,
vllm/entrypoints/launchers/app.py
-4
View File
@@ -8,7 +8,6 @@ from fastapi import FastAPI
from vllm.config import ModelConfig
from vllm.entrypoints.serve.exception_handling.register import init_exception_handler
from vllm.entrypoints.serve.middleware.register import init_entrypoints_middleware
from vllm.plugins.endpoint_plugins.interface import attach_endpoint_plugins
from vllm.tasks import FALLBACK_SUPPORTED_TASKS, SupportedTask
from .api_server.routers import register_api_routers
@@ -41,9 +40,6 @@ def build_app(
register_api_routers(args, app, supported_tasks, model_config)
# Endpoint plugin hooks run after core routers so plugins can inspect them.
attach_endpoint_plugins(app, supported_tasks)
init_exception_handler(app)
init_entrypoints_middleware(args, app, supported_tasks)
return app
vllm/entrypoints/launchers/cli_args.py
+1 -59
View File
@@ -7,13 +7,11 @@ It is kept in a separate file for documentation purposes.
import argparse
import json
import ssl
from collections.abc import Sequence
from dataclasses import field
from typing import Any, Literal
from vllm.config import config
from vllm.engine.arg_utils import AsyncEngineArgs, optional_type
from vllm.entrypoints.openai.models.protocol import LoRAModulePath
from vllm.engine.arg_utils import AsyncEngineArgs
from vllm.logger import init_logger
from vllm.renderers.chat_utils import (
ChatTemplateContentFormatOption,
@@ -31,40 +29,6 @@ from .utils.constants import (
logger = init_logger(__name__)
class LoRAParserAction(argparse.Action):
def __call__(
self,
parser: argparse.ArgumentParser,
namespace: argparse.Namespace,
values: str | Sequence[str] | None,
option_string: str | None = None,
):
if values is None:
values = []
if isinstance(values, str):
raise TypeError("Expected values to be a list")
lora_list: list[LoRAModulePath] = []
for item in values:
if item in [None, ""]: # Skip if item is None or empty string
continue
if "=" in item and "," not in item: # Old format: name=path
name, path = item.split("=", 1)
lora_list.append(LoRAModulePath(name, path))
else: # Assume JSON format
try:
lora_dict = json.loads(item)
lora = LoRAModulePath(**lora_dict)
lora_list.append(lora)
except json.JSONDecodeError:
parser.error(f"Invalid JSON format for --lora-modules: {item}")
except TypeError as e:
parser.error(
f"Invalid fields for --lora-modules: {item} - {str(e)}"
)
setattr(namespace, self.dest, lora_list)
@config
class BaseFrontendArgs:
"""Base arguments for the OpenAI-compatible frontend server.
@@ -74,11 +38,6 @@ class BaseFrontendArgs:
the subclasses.
"""
lora_modules: list[LoRAModulePath] | None = None
"""LoRA modules configurations in either 'name=path' format or JSON format
or JSON list format. Example (old format): `'name=path'` Example (new
format): `{\"name\": \"name\", \"path\": \"lora_path\",
\"base_model_name\": \"id\"}`"""
chat_template: str | None = None
"""The file path to the chat template, or the template in single-line form
for the specified model."""
@@ -170,18 +129,6 @@ class BaseFrontendArgs:
"""
log_error_stack: bool = False
"""If set to True, log the stack trace of error responses"""
tokens_only: bool = False
"""
If set to True, only enable the Tokens In<>Out endpoint.
This is intended for use in a Disaggregated Everything setup.
"""
enable_scale_out: bool = False
"""
If set to True, register the scale-out endpoints (`/inference/v1/generate`
and `/inference/v1/abort_requests`) on `vllm serve`. `vllm serve
--tokens-only` always registers the endpoints it needs regardless of this
flag.
"""
fingerprint_mode: Literal["full", "hash", "custom", "none"] = "full"
"""Controls the ``system_fingerprint`` field on responses.
@@ -208,11 +155,6 @@ class BaseFrontendArgs:
# Special case: default_chat_template_kwargs needs json.loads type
frontend_kwargs["default_chat_template_kwargs"]["type"] = json.loads
# Special case: LoRA modules need custom parser action and
# optional_type(str)
frontend_kwargs["lora_modules"]["type"] = optional_type(str)
frontend_kwargs["lora_modules"]["action"] = LoRAParserAction
# Special case: Tool call parser shows built-in options.
valid_tool_parsers = list(ToolParserManager.list_registered())
parsers_str = ",".join(valid_tool_parsers)

Some files were not shown because too many files have changed in this diff Show More