[MyPy][3/N] Fix MyPy errors in test groups (part 3) (#55939)

Signed-off-by: Taimys <ffy2281693949@gmail.com>
Signed-off-by: Luyi Xiao <81631911+luyixiao95@users.noreply.github.com>
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: Luyi Xiao <81631911+luyixiao95@users.noreply.github.com>
Co-authored-by: Taimys <ffy2281693949@gmail.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Taimys
2026-09-30 11:06:07 +00:00
committed by GitHub
co-authored by Codex Luyi Xiao Taimys Harry Mellor Claude Opus 5.5
parent c4fa6f36a7
commit 5225f1bacb
118 changed files with 948 additions and 605 deletions
+8 -8
View File
@@ -116,7 +116,7 @@ class TestParameterSweep:
def test_from_records_list(self):
"""Test creating ParameterSweep from a list of records."""
records = [
records: list[dict[str, object]] = [
{"max_tokens": 100, "temperature": 0.7},
{"max_tokens": 200, "temperature": 0.9},
]
@@ -127,7 +127,7 @@ class TestParameterSweep:
def test_read_from_dict(self):
"""Test creating ParameterSweep from a dict format."""
data = {
data: dict[str, dict[str, object]] = {
"experiment1": {"max_tokens": 100, "temperature": 0.7},
"experiment2": {"max_tokens": 200, "temperature": 0.9},
}
@@ -149,7 +149,7 @@ class TestParameterSweep:
def test_read_json_list_format(self):
"""Test reading JSON file with list format."""
records = [
records: list[dict[str, object]] = [
{"max_tokens": 100, "temperature": 0.7},
{"max_tokens": 200, "temperature": 0.9},
]
@@ -168,7 +168,7 @@ class TestParameterSweep:
def test_read_json_dict_format(self):
"""Test reading JSON file with dict format."""
data = {
data: dict[str, dict[str, object]] = {
"experiment1": {"max_tokens": 100, "temperature": 0.7},
"experiment2": {"max_tokens": 200, "temperature": 0.9},
}
@@ -190,7 +190,7 @@ class TestParameterSweep:
def test_unique_benchmark_names_validation(self):
"""Test that duplicate _benchmark_name values raise an error."""
# Test with duplicate names in list format
records = [
records: list[dict[str, object]] = [
{"_benchmark_name": "exp1", "max_tokens": 100},
{"_benchmark_name": "exp1", "max_tokens": 200},
]
@@ -200,7 +200,7 @@ class TestParameterSweep:
def test_unique_benchmark_names_multiple_duplicates(self):
"""Test validation with multiple duplicate names."""
records = [
records: list[dict[str, object]] = [
{"_benchmark_name": "exp1", "max_tokens": 100},
{"_benchmark_name": "exp1", "max_tokens": 200},
{"_benchmark_name": "exp2", "max_tokens": 300},
@@ -212,7 +212,7 @@ class TestParameterSweep:
def test_no_benchmark_names_allowed(self):
"""Test that records without _benchmark_name are allowed."""
records = [
records: list[dict[str, object]] = [
{"max_tokens": 100, "temperature": 0.7},
{"max_tokens": 200, "temperature": 0.9},
]
@@ -221,7 +221,7 @@ class TestParameterSweep:
def test_mixed_benchmark_names_allowed(self):
"""Test that mixing records with and without _benchmark_name is allowed."""
records = [
records: list[dict[str, object]] = [
{"_benchmark_name": "exp1", "max_tokens": 100},
{"max_tokens": 200, "temperature": 0.9},
]
+10 -5
View File
@@ -13,6 +13,7 @@ import soundfile as sf
import vllm.benchmarks.datasets.datasets as datasets_module
import vllm.benchmarks.lib.endpoint_request_func as request_func_module
from vllm.benchmarks.lib.endpoint_request_func import RequestFuncInput
from vllm.tokenizers import TokenizerLike
pytestmark = pytest.mark.skip_global_cleanup
@@ -44,6 +45,10 @@ class _CohereNameOnlyTokenizer(_Tokenizer):
super().__init__("cohere/some-local-checkpoint")
def _as_tokenizer(tokenizer: _Tokenizer) -> TokenizerLike:
return cast(TokenizerLike, tokenizer)
def _write_wav(path: Path, duration_s: float = 0.1, sample_rate: int = 16_000) -> None:
num_samples = int(duration_s * sample_rate)
sf.write(path, np.zeros(num_samples, dtype=np.float32), sample_rate)
@@ -109,7 +114,7 @@ def test_asr_dataset_sample_handles_local_audio_paths(tmp_path: Path) -> None:
]
samples = dataset.sample(
tokenizer=_Tokenizer(),
tokenizer=_as_tokenizer(_Tokenizer()),
num_requests=1,
output_len=32,
asr_min_audio_len_sec=0.0,
@@ -146,7 +151,7 @@ def test_asr_dataset_sample_handles_embedded_audio_bytes(
]
samples = dataset.sample(
tokenizer=_Tokenizer(),
tokenizer=_as_tokenizer(_Tokenizer()),
num_requests=1,
output_len=32,
asr_min_audio_len_sec=0.0,
@@ -240,7 +245,7 @@ def _make_asr_dataset(tmp_path: Path) -> datasets_module.ASRDataset:
def test_asr_dataset_cohere_class_name_gets_decoder_prompt(tmp_path: Path) -> None:
dataset = _make_asr_dataset(tmp_path)
samples = dataset.sample(
tokenizer=CohereAsrTokenizer(),
tokenizer=_as_tokenizer(CohereAsrTokenizer()),
num_requests=1,
output_len=32,
asr_min_audio_len_sec=0.0,
@@ -255,7 +260,7 @@ def test_asr_dataset_cohere_name_or_path_fallback_gets_decoder_prompt(
) -> None:
dataset = _make_asr_dataset(tmp_path)
samples = dataset.sample(
tokenizer=_CohereNameOnlyTokenizer(),
tokenizer=_as_tokenizer(_CohereNameOnlyTokenizer()),
num_requests=1,
output_len=32,
asr_min_audio_len_sec=0.0,
@@ -268,7 +273,7 @@ def test_asr_dataset_cohere_name_or_path_fallback_gets_decoder_prompt(
def test_asr_dataset_unknown_tokenizer_gets_empty_prompt(tmp_path: Path) -> None:
dataset = _make_asr_dataset(tmp_path)
samples = dataset.sample(
tokenizer=_Tokenizer(name_or_path="some-other/asr-model"),
tokenizer=_as_tokenizer(_Tokenizer(name_or_path="some-other/asr-model")),
num_requests=1,
output_len=32,
asr_min_audio_len_sec=0.0,
+2 -2
View File
@@ -271,7 +271,7 @@ def test_bfcl_prompt_len_includes_tools(tmp_path: Path) -> None:
args.num_prompts = 1
with _patch_hf_api(fake_download):
samples = get_samples(args, fake)
samples = get_samples(args, fake) # type: ignore[arg-type]
assert len(samples) == 1
assert captured["tools"] is not None, (
@@ -311,7 +311,7 @@ def test_bfcl_prompt_len_falls_back_when_tokenizer_rejects_tools(
args.num_prompts = 1
with _patch_hf_api(fake_download):
samples = get_samples(args, _LegacyTokenizer())
samples = get_samples(args, _LegacyTokenizer()) # type: ignore[arg-type]
assert len(samples) == 1
assert samples[0].prompt_len > 0
@@ -58,7 +58,7 @@ def test_chat_template_kwargs_forwarded(tmp_path: Path) -> None:
_write_one(jsonl)
tok = _RecordingTokenizer()
get_samples(_args(str(jsonl), {"thinking": True}), tok)
get_samples(_args(str(jsonl), {"thinking": True}), tok) # type: ignore[arg-type]
assert tok.captured_kwargs == {"thinking": True}
@@ -70,6 +70,6 @@ def test_chat_template_kwargs_default_is_noop(tmp_path: Path) -> None:
_write_one(jsonl)
tok = _RecordingTokenizer()
get_samples(_args(str(jsonl), None), tok)
get_samples(_args(str(jsonl), None), tok) # type: ignore[arg-type]
assert tok.captured_kwargs == {}
+29 -15
View File
@@ -4,7 +4,7 @@ import json
from argparse import Namespace
from io import BytesIO
from pathlib import Path
from typing import Any
from typing import Any, cast
import pybase64 as base64
import pytest
@@ -17,6 +17,7 @@ from vllm.benchmarks.lib.endpoint_request_func import (
_get_chat_content,
_get_chat_messages,
)
from vllm.tokenizers import TokenizerLike
pytestmark = pytest.mark.skip_global_cleanup
@@ -31,6 +32,10 @@ class _Tokenizer:
return _TokenizedPrompt(prompt)
def _tokenizer() -> TokenizerLike:
return cast(TokenizerLike, _Tokenizer())
def _write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None:
with path.open("w") as f:
for row in rows:
@@ -92,7 +97,7 @@ def test_get_samples_custom_image_cli_path_supports_multi_image_and_content(
],
)
samples = get_samples(_args_for_custom_image(jsonl), _Tokenizer())
samples = get_samples(_args_for_custom_image(jsonl), _tokenizer())
assert len(samples) == 2
assert samples[0].request_id == "req-0"
@@ -106,6 +111,7 @@ def test_get_samples_custom_image_cli_path_supports_multi_image_and_content(
assert samples[1].multi_modal_data is None
assert isinstance(samples[1].prompt, list)
assert samples[1].prompt[0] == {"type": "text", "text": "Now compare "}
assert isinstance(samples[1].prompt[1], dict)
assert samples[1].prompt[1]["image_url"]["url"] == f"file://{image_c}"
@@ -126,7 +132,7 @@ def test_custom_image_dataset_uses_all_image_files(tmp_path: Path) -> None:
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
samples = dataset.sample(
tokenizer=_Tokenizer(),
tokenizer=_tokenizer(),
num_requests=1,
output_len=32,
)
@@ -171,7 +177,7 @@ def test_custom_image_dataset_preserves_interleaved_content_order(
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
samples = dataset.sample(
tokenizer=_Tokenizer(),
tokenizer=_tokenizer(),
num_requests=1,
output_len=32,
)
@@ -181,14 +187,18 @@ def test_custom_image_dataset_preserves_interleaved_content_order(
assert sample.multi_modal_data is None
assert sample.prompt_len == 2
assert isinstance(sample.prompt, list)
assert [part["type"] for part in sample.prompt] == [
parts = []
for part in sample.prompt:
assert isinstance(part, dict)
parts.append(part)
assert [part["type"] for part in parts] == [
"text",
"image_url",
"text",
"image_url",
]
assert sample.prompt[1]["image_url"]["url"] == f"file://{image_a}"
assert sample.prompt[3]["image_url"] == {
assert parts[1]["image_url"]["url"] == f"file://{image_a}"
assert parts[3]["image_url"] == {
"url": f"file://{image_b}",
"detail": "low",
}
@@ -223,7 +233,7 @@ def test_custom_image_dataset_wraps_interleaved_content_for_multimodal_chat(
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
samples = dataset.sample(
tokenizer=_Tokenizer(),
tokenizer=_tokenizer(),
num_requests=1,
output_len=32,
enable_multimodal_chat=True,
@@ -289,7 +299,7 @@ def test_custom_image_dataset_encodes_image_media_when_requested(
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
samples = dataset.sample(
tokenizer=_Tokenizer(),
tokenizer=_tokenizer(),
num_requests=1,
output_len=32,
ensure_client_side_data=True,
@@ -335,7 +345,7 @@ def test_custom_image_dataset_encodes_interleaved_image_media(
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
samples = dataset.sample(
tokenizer=_Tokenizer(),
tokenizer=_tokenizer(),
num_requests=1,
output_len=32,
ensure_client_side_data=True,
@@ -343,9 +353,13 @@ def test_custom_image_dataset_encodes_interleaved_image_media(
sample = samples[0]
assert isinstance(sample.prompt, list)
_assert_png_data_url(sample.prompt[1]["image_url"]["url"])
_assert_png_data_url(sample.prompt[2]["image_url"]["url"])
assert sample.prompt[2]["image_url"]["detail"] == "low"
parts = []
for part in sample.prompt:
assert isinstance(part, dict)
parts.append(part)
_assert_png_data_url(parts[1]["image_url"]["url"])
_assert_png_data_url(parts[2]["image_url"]["url"])
assert parts[2]["image_url"]["detail"] == "low"
@pytest.mark.benchmark
@@ -363,7 +377,7 @@ def test_custom_image_dataset_rejects_invalid_image_media(
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
with pytest.raises(ValueError, match="Invalid image URL"):
dataset.sample(
tokenizer=_Tokenizer(),
tokenizer=_tokenizer(),
num_requests=1,
output_len=32,
ensure_client_side_data=True,
@@ -380,7 +394,7 @@ def test_custom_image_dataset_rejects_invalid_content_part(
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
with pytest.raises(ValueError, match="type 'text', 'image', or 'image_url'"):
dataset.sample(
tokenizer=_Tokenizer(),
tokenizer=_tokenizer(),
num_requests=1,
output_len=32,
)
+2
View File
@@ -37,6 +37,7 @@ def random_dataset_params() -> Params:
def _fingerprint_sample(req: SampleRequest) -> tuple[str, int, int]:
"""Project a SampleRequest into a comparable tuple."""
assert isinstance(req.prompt, str)
return (req.prompt, req.prompt_len, req.expected_output_len)
@@ -153,6 +154,7 @@ def _mm_fingerprint_sample(
- count of multimodal items
- per-item type and URL prefix (e.g., 'data:image/jpeg;base64,')
"""
assert isinstance(req.prompt, str)
items = req.multi_modal_data or []
item_prefixes: list[str] = []
for it in items:
+4 -4
View File
@@ -19,7 +19,7 @@ class _FakeTokenizer(TokenizerLike):
# -- Properties required by TokenizerLike --
@classmethod
def from_pretrained(cls, path_or_repo_id, *a, **kw): # type: ignore[override]
def from_pretrained(cls, path_or_repo_id, *a, **kw):
return cls()
@property
@@ -80,16 +80,16 @@ class _FakeTokenizer(TokenizerLike):
def apply_chat_template(self, messages, **kw): # type: ignore[override]
raise NotImplementedError
def convert_tokens_to_ids(self, tokens): # type: ignore[override]
def convert_tokens_to_ids(self, tokens):
raise NotImplementedError
def convert_tokens_to_string(self, tokens: list[str]) -> str:
raise NotImplementedError
def decode(self, ids, skip_special_tokens: bool = False) -> str: # type: ignore[override]
def decode(self, ids, skip_special_tokens: bool = False) -> str:
raise NotImplementedError
def convert_ids_to_tokens( # type: ignore[override]
def convert_ids_to_tokens(
self, ids, skip_special_tokens: bool = False
) -> list[str]:
raise NotImplementedError
+3 -2
View File
@@ -9,6 +9,7 @@ serve-side dataset coverage.
"""
import subprocess
from argparse import Namespace
from types import SimpleNamespace
import pytest
@@ -289,7 +290,7 @@ def test_to_serve_args_supports_mm_processor_flags() -> None:
explicit 0 instead of coercing it to None (which later crashed
RandomMultiModalDataset.get_prefix on ``None <= 0``).
"""
args = SimpleNamespace(
args = Namespace(
random_input_len=32,
random_output_len=1,
random_prefix_len=0,
@@ -306,7 +307,7 @@ def test_to_serve_args_supports_mm_processor_flags() -> None:
def test_to_serve_args_keeps_legacy_flag_fallback() -> None:
"""Callers that only set the legacy flags still get them mapped."""
args = SimpleNamespace(
args = Namespace(
input_len=256,
output_len=8,
prefix_len=16,
@@ -355,9 +355,11 @@ class TestToolResultContent:
if m["role"] == "user" and isinstance(m.get("content"), list)
]
assert len(follow_up) == 1
assert follow_up[0]["content"][0]["image_url"]["url"] == (
"data:image/jpeg;base64,QUFB"
)
content = follow_up[0]["content"]
assert isinstance(content, list)
image = content[0]
assert image["type"] == "image_url"
assert image["image_url"]["url"] == "data:image/jpeg;base64,QUFB"
def test_tool_result_with_multiple_images(self):
request = self._make_tool_result_request(
@@ -387,7 +389,12 @@ class TestToolResultContent:
if m["role"] == "user" and isinstance(m.get("content"), list)
]
assert len(follow_up) == 1
urls = [p["image_url"]["url"] for p in follow_up[0]["content"]]
content = follow_up[0]["content"]
assert isinstance(content, list)
urls = []
for part in content:
assert part["type"] == "image_url"
urls.append(part["image_url"]["url"])
assert urls == [
"data:image/png;base64,IMG1",
"https://example.com/img2.jpg",
@@ -584,7 +591,8 @@ class TestThinkingBlockConversion:
asst = asst_msgs[0]
assert asst.get("reasoning") == "I need to call the calculator."
tool_calls = list(asst.get("tool_calls", []))
tool_calls = asst.get("tool_calls")
assert tool_calls is not None
assert len(tool_calls) == 1
assert tool_calls[0]["function"]["name"] == "calculator"
# No text content alongside reasoning + tool_use.
@@ -1439,7 +1447,7 @@ class TestDetectMergeInlineSystem:
model_config=None,
)
assert (
AnthropicServingMessages._should_merge_inline_system(online_renderer)
AnthropicServingMessages._should_merge_inline_system(online_renderer) # type: ignore[arg-type]
is expected
)
assert mock_logger.warning_once.called is expected
@@ -7,22 +7,26 @@ from unittest.mock import Mock
import pytest
from vllm import LLM, SamplingParams
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
from vllm.inputs import TextPrompt
from vllm.outputs import RequestOutput
def _make_mock_llm() -> LLM:
llm = object.__new__(LLM)
llm.model_config = SimpleNamespace(
llm.model_config = SimpleNamespace( # type: ignore[assignment]
runner_type="generate", enable_prompt_embeds=False
)
return llm
def test_generate_forwards_mm_processor_kwargs() -> None:
def test_generate_forwards_mm_processor_kwargs(monkeypatch: pytest.MonkeyPatch) -> None:
llm = _make_mock_llm()
mm_processor_kwargs = {"num_crops": 4}
sampling_params = SamplingParams(max_tokens=1)
llm._run_completion = Mock(return_value=["ok"])
_run_completion = Mock(return_value=["ok"])
monkeypatch.setattr(llm, "_run_completion", _run_completion)
outputs = llm.generate(
"prompt",
@@ -31,17 +35,18 @@ def test_generate_forwards_mm_processor_kwargs() -> None:
)
assert outputs == ["ok"]
assert llm._run_completion.call_args.kwargs["mm_processor_kwargs"] == (
assert _run_completion.call_args.kwargs["mm_processor_kwargs"] == (
mm_processor_kwargs
)
def test_enqueue_forwards_mm_processor_kwargs() -> None:
def test_enqueue_forwards_mm_processor_kwargs(monkeypatch: pytest.MonkeyPatch) -> None:
llm = _make_mock_llm()
mm_processor_kwargs = {"do_resize": False}
sampling_params = SamplingParams(max_tokens=1)
llm._add_completion_requests = Mock(return_value=["req-0"])
_add_completion_requests = Mock(return_value=["req-0"])
monkeypatch.setattr(llm, "_add_completion_requests", _add_completion_requests)
request_ids = llm.enqueue(
"prompt",
@@ -51,18 +56,19 @@ def test_enqueue_forwards_mm_processor_kwargs() -> None:
)
assert request_ids == ["req-0"]
assert llm._add_completion_requests.call_args.kwargs["mm_processor_kwargs"] == (
assert _add_completion_requests.call_args.kwargs["mm_processor_kwargs"] == (
mm_processor_kwargs
)
def test_chat_forwards_mm_processor_kwargs() -> None:
def test_chat_forwards_mm_processor_kwargs(monkeypatch: pytest.MonkeyPatch) -> None:
llm = _make_mock_llm()
mm_processor_kwargs = {"do_pan_and_scan": True}
sampling_params = SamplingParams(max_tokens=1)
messages = [{"role": "user", "content": "hello"}]
messages: list[ChatCompletionMessageParam] = [{"role": "user", "content": "hello"}]
llm._run_chat = Mock(return_value=["ok"])
_run_chat = Mock(return_value=["ok"])
monkeypatch.setattr(llm, "_run_chat", _run_chat)
outputs = llm.chat(
messages,
@@ -71,18 +77,19 @@ def test_chat_forwards_mm_processor_kwargs() -> None:
)
assert outputs == ["ok"]
assert llm._run_chat.call_args.kwargs["mm_processor_kwargs"] == (
mm_processor_kwargs
)
assert _run_chat.call_args.kwargs["mm_processor_kwargs"] == mm_processor_kwargs
def test_enqueue_chat_forwards_mm_processor_kwargs() -> None:
def test_enqueue_chat_forwards_mm_processor_kwargs(
monkeypatch: pytest.MonkeyPatch,
) -> None:
llm = _make_mock_llm()
mm_processor_kwargs = {"do_pan_and_scan": True}
sampling_params = SamplingParams(max_tokens=1)
messages = [{"role": "user", "content": "hello"}]
messages: list[ChatCompletionMessageParam] = [{"role": "user", "content": "hello"}]
llm._add_chat_requests = Mock(return_value=["req-0"])
_add_chat_requests = Mock(return_value=["req-0"])
monkeypatch.setattr(llm, "_add_chat_requests", _add_chat_requests)
request_ids = llm.enqueue_chat(
messages,
@@ -92,67 +99,79 @@ def test_enqueue_chat_forwards_mm_processor_kwargs() -> None:
)
assert request_ids == ["req-0"]
assert llm._add_chat_requests.call_args.kwargs["mm_processor_kwargs"] == (
assert _add_chat_requests.call_args.kwargs["mm_processor_kwargs"] == (
mm_processor_kwargs
)
def test_run_chat_forwards_mm_processor_kwargs() -> None:
def test_run_chat_forwards_mm_processor_kwargs(monkeypatch: pytest.MonkeyPatch) -> None:
llm = _make_mock_llm()
mm_processor_kwargs = {"num_crops": 8}
sampling_params = SamplingParams(max_tokens=1)
messages = [{"role": "user", "content": "hello"}]
messages: list[ChatCompletionMessageParam] = [{"role": "user", "content": "hello"}]
sentinel_output = ["done"]
llm._add_chat_requests = Mock()
llm._run_engine = Mock(return_value=sentinel_output)
_add_chat_requests = Mock()
monkeypatch.setattr(llm, "_add_chat_requests", _add_chat_requests)
_run_engine = Mock(return_value=sentinel_output)
monkeypatch.setattr(llm, "_run_engine", _run_engine)
outputs = llm._run_chat(
messages=messages,
params=sampling_params,
output_type=object,
output_type=RequestOutput,
use_tqdm=False,
mm_processor_kwargs=mm_processor_kwargs,
)
assert outputs == sentinel_output
assert llm._add_chat_requests.call_args.kwargs["mm_processor_kwargs"] == (
assert _add_chat_requests.call_args.kwargs["mm_processor_kwargs"] == (
mm_processor_kwargs
)
def test_run_completion_forwards_mm_processor_kwargs() -> None:
def test_run_completion_forwards_mm_processor_kwargs(
monkeypatch: pytest.MonkeyPatch,
) -> None:
llm = _make_mock_llm()
mm_processor_kwargs = {"min_pixels": 4 * 28 * 28}
sampling_params = SamplingParams(max_tokens=1)
sentinel_output = ["done"]
llm._add_completion_requests = Mock()
llm._run_engine = Mock(return_value=sentinel_output)
_add_completion_requests = Mock()
monkeypatch.setattr(llm, "_add_completion_requests", _add_completion_requests)
_run_engine = Mock(return_value=sentinel_output)
monkeypatch.setattr(llm, "_run_engine", _run_engine)
outputs = llm._run_completion(
prompts=["prompt"],
params=sampling_params,
output_type=object,
output_type=RequestOutput,
use_tqdm=False,
mm_processor_kwargs=mm_processor_kwargs,
)
assert outputs == sentinel_output
assert llm._add_completion_requests.call_args.kwargs["mm_processor_kwargs"] == (
assert _add_completion_requests.call_args.kwargs["mm_processor_kwargs"] == (
mm_processor_kwargs
)
def test_add_completion_requests_forwards_mm_processor_kwargs() -> None:
def test_add_completion_requests_forwards_mm_processor_kwargs(
monkeypatch: pytest.MonkeyPatch,
) -> None:
llm = _make_mock_llm()
mm_processor_kwargs = {"max_dynamic_patch": 4}
sampling_params = SamplingParams(max_tokens=1)
llm._params_to_seq = Mock(return_value=[sampling_params])
llm._lora_request_to_seq = Mock(return_value=[None])
llm._priority_to_seq = Mock(return_value=[0])
llm._preprocess_cmpl_one = Mock(return_value={"prompt_token_ids": [1]})
_params_to_seq = Mock(return_value=[sampling_params])
monkeypatch.setattr(llm, "_params_to_seq", _params_to_seq)
_lora_request_to_seq = Mock(return_value=[None])
monkeypatch.setattr(llm, "_lora_request_to_seq", _lora_request_to_seq)
_priority_to_seq = Mock(return_value=[0])
monkeypatch.setattr(llm, "_priority_to_seq", _priority_to_seq)
_preprocess_cmpl_one = Mock(return_value={"prompt_token_ids": [1]})
monkeypatch.setattr(llm, "_preprocess_cmpl_one", _preprocess_cmpl_one)
captured_prompts = []
@@ -160,7 +179,8 @@ def test_add_completion_requests_forwards_mm_processor_kwargs() -> None:
captured_prompts.extend(prompts)
return ["req-0"]
llm._render_and_add_requests = Mock(side_effect=fake_render_and_add_requests)
_render_and_add_requests = Mock(side_effect=fake_render_and_add_requests)
monkeypatch.setattr(llm, "_render_and_add_requests", _render_and_add_requests)
request_ids = llm._add_completion_requests(
prompts=["prompt"],
@@ -170,7 +190,7 @@ def test_add_completion_requests_forwards_mm_processor_kwargs() -> None:
)
assert request_ids == ["req-0"]
llm._preprocess_cmpl_one.assert_called_once_with(
_preprocess_cmpl_one.assert_called_once_with(
"prompt",
None,
mm_processor_kwargs=mm_processor_kwargs,
@@ -183,7 +203,7 @@ def test_preprocess_cmpl_applies_mm_processor_kwargs_to_renderer(
) -> None:
llm = _make_mock_llm()
mm_processor_kwargs = {"num_crops": 8}
prompt = {"prompt": "<image>", "multi_modal_data": {"image": object()}}
prompt: TextPrompt = {"prompt": "<image>", "multi_modal_data": {"image": object()}}
renderer = Mock()
renderer.default_cmpl_tok_params = Mock()
@@ -213,7 +233,7 @@ def test_preprocess_cmpl_keeps_prompt_mm_processor_kwargs_when_no_override(
monkeypatch: pytest.MonkeyPatch,
) -> None:
llm = _make_mock_llm()
prompt = {
prompt: TextPrompt = {
"prompt": "<image>",
"multi_modal_data": {"image": object()},
"mm_processor_kwargs": {"num_crops": 2},
@@ -243,7 +263,9 @@ def test_preprocess_cmpl_keeps_prompt_mm_processor_kwargs_when_no_override(
def test_preprocess_chat_applies_mm_processor_kwargs_to_renderer() -> None:
llm = _make_mock_llm()
mm_processor_kwargs = {"num_crops": 8}
messages = [[{"role": "user", "content": "Describe this image."}]]
messages: list[list[ChatCompletionMessageParam]] = [
[{"role": "user", "content": "Describe this image."}]
]
renderer = Mock()
renderer.tokenizer = object()
@@ -269,7 +291,9 @@ def test_preprocess_chat_applies_mm_processor_kwargs_to_renderer() -> None:
def test_preprocess_chat_omits_mm_processor_kwargs_when_no_override() -> None:
llm = _make_mock_llm()
messages = [[{"role": "user", "content": "Describe this image."}]]
messages: list[list[ChatCompletionMessageParam]] = [
[{"role": "user", "content": "Describe this image."}]
]
renderer = Mock()
renderer.tokenizer = object()
@@ -291,7 +315,9 @@ def test_preprocess_chat_omits_mm_processor_kwargs_when_no_override() -> None:
def test_preprocess_chat_defaults_add_special_tokens_to_false() -> None:
# Matches `ChatCompletionRequest.add_special_tokens` on the server.
llm = _make_mock_llm()
messages = [[{"role": "user", "content": "hi"}]]
messages: list[list[ChatCompletionMessageParam]] = [
[{"role": "user", "content": "hi"}]
]
renderer = Mock()
renderer.tokenizer = object()
@@ -309,7 +335,9 @@ def test_preprocess_chat_defaults_add_special_tokens_to_false() -> None:
def test_preprocess_chat_tokenization_kwargs_override_add_special_tokens() -> None:
llm = _make_mock_llm()
messages = [[{"role": "user", "content": "hi"}]]
messages: list[list[ChatCompletionMessageParam]] = [
[{"role": "user", "content": "hi"}]
]
renderer = Mock()
renderer.tokenizer = object()
@@ -47,18 +47,20 @@ async def client(server):
@pytest.fixture(scope="session")
def base64_encoded_audio() -> dict[str, str]:
return {
audio_url: encode_audio_base64(*fetch_audio(audio_url))
for audio_url in TEST_AUDIO_URLS
}
encoded = {}
for audio_url in TEST_AUDIO_URLS:
audio, sample_rate = fetch_audio(audio_url)
encoded[audio_url] = encode_audio_base64(audio, int(sample_rate))
return encoded
@pytest.fixture(scope="session")
def url_encoded_audio() -> dict[str, str]:
return {
audio_url: encode_audio_url(*fetch_audio(audio_url))
for audio_url in TEST_AUDIO_URLS
}
encoded = {}
for audio_url in TEST_AUDIO_URLS:
audio, sample_rate = fetch_audio(audio_url)
encoded[audio_url] = encode_audio_url(audio, int(sample_rate))
return encoded
def dummy_messages_from_audio_url(
@@ -7,6 +7,7 @@ import torch
from tests.models.utils import softmax
from vllm import LLM, ClassificationRequestOutput, PoolingParams
from vllm.inputs import PromptType
from vllm.tasks import PoolingTask
MODEL_NAME = "jason9693/Qwen2.5-1.5B-apeach"
@@ -53,7 +54,8 @@ def test_token_ids_prompts(llm: LLM):
@pytest.mark.skip_global_cleanup
def test_list_prompts(llm: LLM):
outputs = llm.classify([prompt, prompt_token_ids], use_tqdm=False)
prompts: list[PromptType] = [prompt, prompt_token_ids]
outputs = llm.classify(prompts, use_tqdm=False)
assert len(outputs) == 2
for i in range(len(outputs)):
assert isinstance(outputs[i], ClassificationRequestOutput)
@@ -21,6 +21,7 @@ from vllm.entrypoints.pooling.typing import (
PoolingEngineInput,
PoolingServeContext,
)
from vllm.inputs import tokens_input
from vllm.outputs import PoolingOutput, PoolingRequestOutput
@@ -28,7 +29,7 @@ class TestEmbeddingRequestParsing:
"""Unit tests for OpenAI embedding request parsing."""
def test_input_messages_parses_as_chat_request(self):
request = TypeAdapter(EmbeddingRequest).validate_python(
request: EmbeddingRequest = TypeAdapter(EmbeddingRequest).validate_python(
{
"model": "test",
"input": [{"role": "user", "content": "hello"}],
@@ -42,7 +43,7 @@ class TestEmbeddingRequestParsing:
assert request.chat_template_kwargs == {"instruction": "Represent the query: "}
def test_batched_input_messages_parses_as_batch_chat_input_request(self):
request = TypeAdapter(EmbeddingRequest).validate_python(
request: EmbeddingRequest = TypeAdapter(EmbeddingRequest).validate_python(
{
"model": "test",
"input": [
@@ -65,7 +66,7 @@ class TestEmbeddingRequestParsing:
assert request.chat_template_kwargs == {"instruction": "Represent the query: "}
def test_token_ids_still_parse_as_completion_request(self):
request = TypeAdapter(EmbeddingRequest).validate_python(
request: EmbeddingRequest = TypeAdapter(EmbeddingRequest).validate_python(
{
"model": "test",
"input": [[1, 2, 3], [4, 5]],
@@ -76,7 +77,7 @@ class TestEmbeddingRequestParsing:
assert request.input == [[1, 2, 3], [4, 5]]
def test_messages_still_parses_as_chat_request(self):
request = TypeAdapter(EmbeddingRequest).validate_python(
request: EmbeddingRequest = TypeAdapter(EmbeddingRequest).validate_python(
{
"model": "test",
"messages": [{"role": "user", "content": "hello"}],
@@ -89,7 +90,7 @@ class TestEmbeddingRequestParsing:
assert request.chat_template_kwargs == {"instruction": "Represent the query: "}
def test_batched_messages_parses_as_batch_chat_request(self):
request = TypeAdapter(EmbeddingRequest).validate_python(
request: EmbeddingRequest = TypeAdapter(EmbeddingRequest).validate_python(
{
"model": "test",
"messages": [
@@ -410,13 +411,13 @@ class TestChunkedEmbeddingProcessing:
@classmethod
def _make_handler(cls):
handler = object.__new__(EmbedIOProcessor)
handler.model_config = cls._FakeModelConfig()
handler.model_config = cls._FakeModelConfig() # type: ignore[assignment]
handler.enable_chunked_processing = True
return handler
@staticmethod
def _make_context() -> PoolingServeContext[EmbeddingCompletionRequest]:
request = TypeAdapter(EmbeddingRequest).validate_python(
request: EmbeddingRequest = TypeAdapter(EmbeddingRequest).validate_python(
{
"model": "test",
"input": [[0, 1, 2, 3, 4], [10, 11]],
@@ -431,13 +432,13 @@ class TestChunkedEmbeddingProcessing:
request_id="embd-client-prompt-999-chunk-888",
engine_inputs=[
PoolingEngineInput(
prompts={"prompt_token_ids": [0, 1, 2, 3, 4]},
prompts=tokens_input(prompt_token_ids=[0, 1, 2, 3, 4]),
params=pooling_params,
lora_requests=None,
priorities=0,
),
PoolingEngineInput(
prompts={"prompt_token_ids": [10, 11]},
prompts=tokens_input(prompt_token_ids=[10, 11]),
params=pooling_params,
lora_requests=None,
priorities=0,
@@ -7,6 +7,7 @@ import torch
import torch.nn.functional as F
from vllm import LLM, EmbeddingRequestOutput, PoolingParams
from vllm.inputs import PromptType
from vllm.tasks import PoolingTask
MODEL_NAME = "intfloat/multilingual-e5-small"
@@ -54,7 +55,8 @@ def test_token_ids_prompts(llm: LLM):
@pytest.mark.skip_global_cleanup
def test_list_prompts(llm: LLM):
outputs = llm.embed([prompt, prompt_token_ids], use_tqdm=False)
prompts: list[PromptType] = [prompt, prompt_token_ids]
outputs = llm.embed(prompts, use_tqdm=False)
assert len(outputs) == 2
for i in range(len(outputs)):
assert isinstance(outputs[i], EmbeddingRequestOutput)
+13 -9
View File
@@ -12,9 +12,13 @@ import requests
import torch
import torch.nn.functional as F
from tests.models.language.pooling.embed_utils import run_embedding_correctness_test
from tests.models.language.pooling.embed_utils import (
float_embeddings,
run_embedding_correctness_test,
)
from tests.models.utils import check_embeddings_close
from tests.utils import RemoteOpenAIServer
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
from vllm.entrypoints.pooling.embed.protocol import EmbeddingResponse
from vllm.entrypoints.pooling.pooling.protocol import PoolingResponse
from vllm.entrypoints.pooling.utils import (
@@ -145,7 +149,7 @@ async def test_completion_request(
assert embeddings.usage.prompt_tokens == len(input_tokens)
assert embeddings.usage.total_tokens == len(input_tokens)
vllm_outputs = [d.embedding for d in embeddings.data]
vllm_outputs = float_embeddings(embeddings)
run_embedding_correctness_test(hf_model, [input_text], vllm_outputs)
# test input: list[int]
@@ -165,7 +169,7 @@ async def test_completion_request(
assert embeddings.usage.prompt_tokens == len(input_tokens)
assert embeddings.usage.total_tokens == len(input_tokens)
vllm_outputs = [d.embedding for d in embeddings.data]
vllm_outputs = float_embeddings(embeddings)
run_embedding_correctness_test(hf_model, [input_text], vllm_outputs)
@@ -194,7 +198,7 @@ async def test_completion_request_batched(
assert embeddings.usage.prompt_tokens == len(input_tokens) * N
assert embeddings.usage.total_tokens == len(input_tokens) * N
vllm_outputs = [d.embedding for d in embeddings.data]
vllm_outputs = float_embeddings(embeddings)
run_embedding_correctness_test(hf_model, input_texts, vllm_outputs)
# test list[list[int]]
@@ -214,7 +218,7 @@ async def test_completion_request_batched(
assert embeddings.usage.prompt_tokens == len(input_tokens) * N
assert embeddings.usage.total_tokens == len(input_tokens) * N
vllm_outputs = [d.embedding for d in embeddings.data]
vllm_outputs = float_embeddings(embeddings)
run_embedding_correctness_test(hf_model, input_texts, vllm_outputs)
@@ -331,7 +335,7 @@ async def test_padding(client: openai.AsyncOpenAI, model_name: str):
async def test_chat_request(
server: RemoteOpenAIServer, client: openai.AsyncOpenAI, model_name: str
):
messages = [
messages: list[ChatCompletionMessageParam] = [
{
"role": "user",
"content": "The cat sat on the mat.",
@@ -382,8 +386,8 @@ async def test_chat_request(
assert chat_embeddings.created <= completion_embeddings.created
# Use tolerance-based comparison for embeddings
check_embeddings_close(
embeddings_0_lst=[d.embedding for d in chat_embeddings.data],
embeddings_1_lst=[d.embedding for d in completion_embeddings.data],
embeddings_0_lst=float_embeddings(chat_embeddings),
embeddings_1_lst=float_embeddings(completion_embeddings),
name_0="chat",
name_1="completion",
)
@@ -488,7 +492,7 @@ async def test_invocations_completion_request(
@pytest.mark.asyncio
async def test_invocations_chat_request(server: RemoteOpenAIServer):
messages = [
messages: list[ChatCompletionMessageParam] = [
{
"role": "user",
"content": "The cat sat on the mat.",
@@ -6,7 +6,10 @@ import openai
import pytest
from tests.conftest import HfRunner
from tests.models.language.pooling.embed_utils import run_embedding_correctness_test
from tests.models.language.pooling.embed_utils import (
float_embeddings,
run_embedding_correctness_test,
)
from tests.models.utils import EmbedModelInfo
from tests.utils import ROCM_EXTRA_ARGS, RemoteOpenAIServer
from vllm.entrypoints.pooling.embed.protocol import EmbeddingResponse
@@ -95,7 +98,7 @@ async def test_matryoshka(
if dimensions is not None:
assert len(embeddings.data[0].embedding) == dimensions
vllm_outputs = [d.embedding for d in embeddings.data]
vllm_outputs = float_embeddings(embeddings)
run_embedding_correctness_test(hf_model, prompts, vllm_outputs, dimensions)
if model_info.is_matryoshka:
@@ -274,6 +274,7 @@ async def test_long_text_embedding_1500_chars(
# Verify embedding vector validity
embedding_vector = embeddings.data[0].embedding
assert isinstance(embedding_vector, list)
assert all(isinstance(x, float) for x in embedding_vector), (
"Embedding vector should contain floats"
)
@@ -322,6 +323,7 @@ async def test_long_text_embedding_2500_chars(
# Verify embedding vector validity
embedding_vector = embeddings.data[0].embedding
assert isinstance(embedding_vector, list)
assert all(isinstance(x, float) for x in embedding_vector), (
"Embedding vector should contain floats"
)
@@ -409,6 +411,7 @@ async def test_chunked_vs_normal_consistency(
# 验证embedding向量的有效性
embedding_vector = embeddings.data[0].embedding
assert isinstance(embedding_vector, list)
assert all(isinstance(x, float) for x in embedding_vector)
assert not all(x == 0 for x in embedding_vector)
@@ -438,6 +441,7 @@ async def test_chunked_processing_response_format(
# Verify embedding vector properties
embedding_vector = embeddings.data[0].embedding
assert isinstance(embedding_vector, list)
import math
vector_norm = math.sqrt(sum(x * x for x in embedding_vector))
@@ -11,6 +11,7 @@ import torch
from tests.models.utils import check_embeddings_close
from tests.utils import RemoteOpenAIServer
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
from vllm.entrypoints.pooling.pooling.protocol import PoolingResponse
from vllm.entrypoints.pooling.utils import (
MetadataItem,
@@ -137,7 +138,7 @@ def test_completion_request_batched(server: RemoteOpenAIServer, model_name: str)
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
async def test_chat_request(server: RemoteOpenAIServer, model_name: str):
messages = [
messages: list[ChatCompletionMessageParam] = [
{
"role": "user",
"content": "The cat sat on the mat.",
@@ -524,7 +525,7 @@ async def test_invocations_chat_request(server: RemoteOpenAIServer):
@pytest.mark.asyncio
async def test_invocations_conversation_chat_request(server: RemoteOpenAIServer):
messages = [
messages: list[ChatCompletionMessageParam] = [
{
"role": "user",
"content": "The cat sat on the mat.",
@@ -47,7 +47,7 @@ def llm_reranker_processor() -> CrossEncoderIOProcessor:
)
processor = CrossEncoderIOProcessor.__new__(CrossEncoderIOProcessor)
processor.model_config = SimpleNamespace(enable_prompt_embeds=False)
processor.model_config = SimpleNamespace(enable_prompt_embeds=False) # type: ignore[assignment]
processor.tokenizer = TokenizersBackend(
tokenizer_object=backend,
unk_token="[UNK]",
@@ -95,12 +95,16 @@ def test_token_type_ids_stay_aligned_with_a_truncated_padded_prompt():
truncation_side="left",
)
prompt = tok_params.apply_post_tokenization(tokenizer, prompt)
processed_prompt = tok_params.apply_post_tokenization(
tokenizer, # type: ignore[arg-type]
prompt,
)
assert "prompt_token_ids" in processed_prompt
token_type_ids = _apply_post_tokenization_to_token_type_ids(
tokenizer, tok_params, token_type_ids
)
prompt_token_ids = prompt["prompt_token_ids"]
prompt_token_ids = processed_prompt["prompt_token_ids"]
assert len(token_type_ids) == len(prompt_token_ids)
# Keeping the last 40 tokens drops the first 10 query tokens, so 10 query
@@ -30,12 +30,18 @@ def test_online_forwards_truncate_prompt_tokens_to_proxy(monkeypatch):
silently disables truncate_prompt_tokens for Jina rerank/score.
"""
proc = JinaRankingIOProcessor.__new__(JinaRankingIOProcessor)
proc.valid_inputs_online = MagicMock(
return_value=ScoringData(data_1=["query"], data_2=["doc"])
monkeypatch.setattr(
proc,
"valid_inputs_online",
MagicMock(return_value=ScoringData(data_1=["query"], data_2=["doc"])),
)
monkeypatch.setattr(proc, "_get_token_limits", MagicMock(return_value=(0, 0)))
monkeypatch.setattr(
proc, "ensure_str", MagicMock(side_effect=lambda data: list(data))
)
monkeypatch.setattr(
proc, "format_docs_prompts_func", MagicMock(return_value="formatted prompt")
)
proc._get_token_limits = MagicMock(return_value=(0, 0))
proc.ensure_str = MagicMock(side_effect=lambda data: list(data))
proc.format_docs_prompts_func = MagicMock(return_value="formatted prompt")
captured: dict[str, object] = {}
@@ -71,8 +77,10 @@ def offline_processor_and_context():
backend.pre_tokenizer = pre_tokenizers.WhitespaceSplit()
proc = JinaRankingIOProcessor.__new__(JinaRankingIOProcessor)
proc.tokenizer = TokenizersBackend(tokenizer_object=backend)
proc.model_config = SimpleNamespace(max_model_len=1024, is_encoder_decoder=False)
proc.renderer = SimpleNamespace(
proc.model_config = SimpleNamespace( # type: ignore[assignment]
max_model_len=1024, is_encoder_decoder=False
)
proc.renderer = SimpleNamespace( # type: ignore[assignment]
default_cmpl_tok_params=TokenizeParams(max_total_tokens=1024)
)
ctx = OfflineScoringInputsContext(
@@ -6,7 +6,7 @@ from unittest.mock import AsyncMock, Mock
import pytest
from vllm.entrypoints.pooling.scoring.serving import ServingScores
from vllm.entrypoints.pooling.typing import PoolingServeContext
from vllm.entrypoints.pooling.typing import PoolingEngineInput, PoolingServeContext
from vllm.inputs import tokens_input
from vllm.pooling_params import PoolingParams
@@ -21,8 +21,18 @@ def _make_context() -> PoolingServeContext:
priorities=None,
prompt_extras=None,
engine_inputs=[
tokens_input(prompt_token_ids=[1]),
tokens_input(prompt_token_ids=[2]),
PoolingEngineInput(
prompts=tokens_input(prompt_token_ids=[1]),
params=PoolingParams(task="token_embed"),
lora_requests=None,
priorities=0,
),
PoolingEngineInput(
prompts=tokens_input(prompt_token_ids=[2]),
params=PoolingParams(task="token_embed"),
lora_requests=None,
priorities=0,
),
],
n_queries=1,
)
@@ -39,10 +49,13 @@ def _query_key(context: PoolingServeContext) -> str:
@pytest.mark.asyncio
async def test_colliding_request_ids_use_distinct_query_cache_keys():
async def test_colliding_request_ids_use_distinct_query_cache_keys(
monkeypatch: pytest.MonkeyPatch,
):
serving = object.__new__(ServingScores)
serving._prepare_generators = AsyncMock()
serving._collect_batch = AsyncMock()
prepare_generators = AsyncMock()
monkeypatch.setattr(serving, "_prepare_generators", prepare_generators)
monkeypatch.setattr(serving, "_collect_batch", AsyncMock())
first = _make_context()
second = _make_context()
@@ -50,9 +63,7 @@ async def test_colliding_request_ids_use_distinct_query_cache_keys():
await serving._flash_late_interaction_encode_queries(context)
await serving._flash_late_interaction_encode_docs(context)
prepared_contexts = [
call.args[0] for call in serving._prepare_generators.await_args_list
]
prepared_contexts = [call.args[0] for call in prepare_generators.await_args_list]
first_query_key = _query_key(prepared_contexts[0])
first_doc_key = _query_key(prepared_contexts[1])
second_query_key = _query_key(prepared_contexts[2])
@@ -53,8 +53,8 @@ def test_rejects_mismatched_lora_requests(processor: PoolingIOProcessor):
def test_rejects_conflicting_pooling_task(processor: PoolingIOProcessor):
processor.model_config = SimpleNamespace(is_encoder_decoder=False)
processor.renderer = SimpleNamespace(
processor.model_config = SimpleNamespace(is_encoder_decoder=False) # type: ignore[assignment]
processor.renderer = SimpleNamespace( # type: ignore[assignment] # Tokenization defaults only.
default_cmpl_tok_params=TokenizeParams(max_total_tokens=None)
)
ctx = OfflineEncodeInputsContext(
@@ -45,6 +45,7 @@ async def test_pooling_token_classify(server: RemoteOpenAIServer, model_name: st
poolings = PoolingResponse.model_validate(response.json())
assert len(poolings.data) == 1
assert len(poolings.data[0].data) == 8
assert isinstance(poolings.data[0].data[0], list)
assert len(poolings.data[0].data[0]) == 2
@@ -71,6 +71,7 @@ async def test_pooling_token_embed(server: RemoteOpenAIServer, model_name: str):
assert poolings.id == f"pooling-{request_id}"
assert len(poolings.data) == 1
assert len(poolings.data[0].data) == len(input_tokens)
assert isinstance(poolings.data[0].data[0], list)
assert len(poolings.data[0].data[0]) == 384
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from contextlib import contextmanager
from typing import Literal
from unittest.mock import MagicMock, patch
import pytest
@@ -14,6 +15,7 @@ from vllm.config import (
VllmConfig,
set_current_vllm_config,
)
from vllm.config.cache import CacheDType
from vllm.platforms import current_platform
from vllm.platforms.cpu import CpuPlatform
from vllm.platforms.interface import DeviceCapability
@@ -21,12 +23,12 @@ from vllm.platforms.interface import DeviceCapability
if current_platform.is_cuda():
from vllm.platforms.cuda import CudaPlatform
else:
CudaPlatform = None
CudaPlatform = None # type: ignore[assignment] # Unavailable platform import.
if current_platform.is_rocm():
from vllm.platforms.rocm import RocmPlatform
else:
RocmPlatform = None
RocmPlatform = None # type: ignore[misc, assignment] # Unavailable platform import.
from vllm.v1.attention.backend import AttentionType
from vllm.v1.attention.backends.registry import AttentionBackendEnum
@@ -354,14 +356,14 @@ def test_invalid_backend():
def test_auto_backend_string(auto_value: str):
"""Test that 'auto' string value triggers automatic backend selection."""
# Using "auto" should result in backend=None (automatic selection)
attention_config = AttentionConfig(backend=auto_value)
attention_config = AttentionConfig(backend=auto_value) # type: ignore[arg-type]
assert attention_config.backend is None
def test_auto_backend_selection_behavior():
"""Test that 'auto' backend behaves same as None (automatic selection)."""
# Create config with explicit "auto"
auto_config = AttentionConfig(backend="auto")
auto_config = AttentionConfig(backend="auto") # type: ignore[arg-type]
# Create config with None (default)
none_config = AttentionConfig(backend=None)
@@ -406,7 +408,7 @@ def test_auto_backend_selection_behavior():
reason="Attention backend FA3 is not supported on ROCm. This test can't succeed.",
)
def test_per_head_quant_scales_backend_selection(
backend_name: str, flash_attn_version: int | None, should_succeed: bool
backend_name: str, flash_attn_version: Literal[2, 3, 4] | None, should_succeed: bool
):
"""Test backend selection when use_per_head_quant_scales=True."""
# Clear cache to ensure fresh backend selection
@@ -559,7 +561,7 @@ def test_non_causal_autoselect_backend():
"int8_per_token_head",
],
)
def test_flash_attn_rejects_unhandled_kv_cache_dtypes(kv_cache_dtype: str):
def test_flash_attn_rejects_unhandled_kv_cache_dtypes(kv_cache_dtype: CacheDType):
"""FlashAttentionBackend must not claim support for kv_cache dtypes
that it cannot handle."""
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
@@ -569,7 +571,7 @@ def test_flash_attn_rejects_unhandled_kv_cache_dtypes(kv_cache_dtype: str):
@pytest.mark.parametrize("kv_cache_dtype", ["fp8", "fp8_e4m3"])
def test_flash_attn_accepts_handled_fp8_variants(
kv_cache_dtype: str, monkeypatch: pytest.MonkeyPatch
kv_cache_dtype: CacheDType, monkeypatch: pytest.MonkeyPatch
):
"""FlashAttentionBackend must accept the two fp8 dtypes it can actually
handle: 'fp8' (alias for fp8_e4m3fn) and 'fp8_e4m3'."""
@@ -876,7 +878,9 @@ def test_rswa_selection_does_not_reuse_causal_result(blackwell_selection):
config = EngineArgs(
model="google/gemma-4-31B-it",
dtype="bfloat16",
attention_config={"backend": "TRITON_FLASHINFER"},
attention_config=AttentionConfig(
backend=AttentionBackendEnum.TRITON_FLASHINFER
),
).create_engine_config()
with set_current_vllm_config(config):
assert (
@@ -151,7 +151,7 @@ def test_cascade(
max_seqlen_k=max_kv_len,
softmax_scale=scale,
causal=True,
window_size=window_size,
window_size=list(window_size),
block_table=block_tables,
softcap=soft_cap if soft_cap is not None else 0,
)
+1 -1
View File
@@ -836,7 +836,7 @@ def varlen_encoder_zentorch_sdpa(
key,
value,
output,
SimpleNamespace(query_start_loc=query_start_loc, causal=False),
SimpleNamespace(query_start_loc=query_start_loc, causal=False), # type: ignore[arg-type]
scale,
sliding_window if sliding_window is not None else -1,
alibi_slopes,
+2 -2
View File
@@ -187,7 +187,7 @@ def test_varlen_with_paged_kv(
max_seqlen_k=max_kv_len,
softmax_scale=scale,
causal=True,
window_size=window_size,
window_size=list(window_size),
block_table=block_tables,
softcap=soft_cap if soft_cap is not None else 0,
fa_version=fa_version,
@@ -274,7 +274,7 @@ def test_fa4_hd256_paged_call_shape(
max_seqlen_k=num_pages * block_size,
softmax_scale=scale,
causal=True,
window_size=window_size,
window_size=list(window_size),
block_table=block_tables[:, :num_pages],
num_splits=1,
fa_version=4,
@@ -197,11 +197,6 @@ def test_decode_concat_ignores_negative_slots_for_cache(cache_format: str) -> No
mixed_cache = initial_cache.clone()
reference_cache = _reference_cache(cache_format, inputs, initial_cache, slots)
scale_inv = torch.ones(1, device="cuda", dtype=torch.float32)
kwargs = {
"ds_mla": cache_format == "fp8_ds_mla",
"q_scale_inv": scale_inv if cache_format == "fp8" else None,
"cache_scale_inv": scale_inv if cache_format == "fp8" else None,
}
output = fused_mla_decode_q_concat_kv_cache_insert(
inputs["ql_nope"],
@@ -210,7 +205,9 @@ def test_decode_concat_ignores_negative_slots_for_cache(cache_format: str) -> No
inputs["k_pe"],
mixed_cache,
slots,
**kwargs,
ds_mla=cache_format == "fp8_ds_mla",
q_scale_inv=scale_inv if cache_format == "fp8" else None,
cache_scale_inv=scale_inv if cache_format == "fp8" else None,
)
expected = _latent_query(inputs["ql_nope"], inputs["q_pe"])
+4 -4
View File
@@ -277,7 +277,7 @@ def test_mha_attn_varlen_forward_flashinfer(
# Override vllm config so get_vit_attn_backend returns FLASHINFER (simulates
# --mm-encoder-attn-backend=FLASHINFER).
vllm_config = get_current_vllm_config()
old_model_config = getattr(vllm_config, "model_config", None)
old_model_config = vllm_config.model_config
minimal_model_config = type(
"MinimalModelConfig",
(),
@@ -376,7 +376,7 @@ def test_mha_attn_varlen_forward_aiter_fp8(
torch.set_default_dtype(dtype)
vllm_config = get_current_vllm_config()
old_model_config = getattr(vllm_config, "model_config", None)
old_model_config = vllm_config.model_config
minimal_model_config = type(
"MinimalModelConfig",
(),
@@ -442,7 +442,7 @@ def test_mha_attn_varlen_forward_aiter_fp8(
)
def test_mha_attn_aiter_fp8_rejects_unsupported_arch(default_vllm_config):
vllm_config = get_current_vllm_config()
old_model_config = getattr(vllm_config, "model_config", None)
old_model_config = vllm_config.model_config
vllm_config.model_config = type(
"MinimalModelConfig",
(),
@@ -468,7 +468,7 @@ def test_mha_attn_aiter_fp8_rejects_unsupported_arch(default_vllm_config):
)
def test_mha_attn_fp8_rejects_wrong_backend(default_vllm_config):
vllm_config = get_current_vllm_config()
old_model_config = getattr(vllm_config, "model_config", None)
old_model_config = vllm_config.model_config
vllm_config.model_config = type(
"MinimalModelConfig",
(),
+17 -12
View File
@@ -1689,8 +1689,10 @@ def test_aiter_sparse_pa_layout_contract(monkeypatch, layout):
)
backend = sparse_attn_mod.MiniMaxM3SparseBackend
assert KVCacheLayout.LHBNC in backend.supported_kv_cache_layouts()
assert KVCacheLayout.LBHNC in backend.supported_kv_cache_layouts()
layouts = backend.supported_kv_cache_layouts()
assert layouts is not None
assert KVCacheLayout.LHBNC in layouts
assert KVCacheLayout.LBHNC in layouts
nb, h = 3, 1
cache_dtype = current_platform.fp8_dtype()
@@ -1772,19 +1774,22 @@ def test_aiter_indexer_requires_the_aiter_attend(monkeypatch):
import vllm.models.minimax_m3.amd.indexer_aiter as indexer_aiter_mod
monkeypatch.setattr(indexer_aiter_mod.current_platform, "is_rocm", lambda: True)
kwargs = dict(
topk_blocks=TOPK,
sparse_block_size=BLOCK_SIZE,
num_index_heads=1,
index_head_dim=HEAD_DIM,
indexer_kv_dtype="fp8_e4m3",
max_model_len=8192,
)
def unsupported_reason() -> str | None:
return indexer_aiter_mod.aiter_indexer_unsupported_reason(
topk_blocks=TOPK,
sparse_block_size=BLOCK_SIZE,
num_index_heads=1,
index_head_dim=HEAD_DIM,
# The implementation also accepts this legacy fp8 alias.
indexer_kv_dtype="fp8_e4m3", # type: ignore[arg-type]
max_model_len=8192,
)
monkeypatch.setattr(
indexer_aiter_mod, "_minimax_m3_aiter_sparse_pa_requested", lambda: False
)
reason = indexer_aiter_mod.aiter_indexer_unsupported_reason(**kwargs)
reason = unsupported_reason()
assert reason is not None and "AITER sparse PA attend" in reason
# With the attend asked for, the gate moves on to the kernel-contract
@@ -1792,7 +1797,7 @@ def test_aiter_indexer_requires_the_aiter_attend(monkeypatch):
monkeypatch.setattr(
indexer_aiter_mod, "_minimax_m3_aiter_sparse_pa_requested", lambda: True
)
reason = indexer_aiter_mod.aiter_indexer_unsupported_reason(**kwargs)
reason = unsupported_reason()
assert reason is None or "AITER sparse PA attend" not in reason
@@ -235,14 +235,14 @@ def test_msa_metadata_builder_prepares_cutlass_for_regular_decode(
builder.num_q_heads = 64
builder.num_kv_heads = 4
builder.topk_blocks = TOPK
builder.kv_cache_spec = SimpleNamespace(num_kv_heads=4)
builder.kv_cache_spec = SimpleNamespace(num_kv_heads=4) # type: ignore[assignment]
builder.kv_cache_dtype = "fp8_e4m3"
builder.decode_backend = "cutlass"
builder.msa_cutlass_plan_cache = object()
builder.msa_cutlass_plan_cache = object() # type: ignore[assignment] # Mocked plan-cache sentinel.
metadata = builder.build(
0,
SimpleNamespace(
SimpleNamespace( # type: ignore[arg-type]
seq_lens_cpu_upper_bound=torch.full((batch,), 257, dtype=torch.int32)
),
)
@@ -377,7 +377,7 @@ def test_query_fp8_stays_valid_when_cutlass_plan_appears_on_replay(
with torch.cuda.stream(stream), override_forward_context(forward_context):
capture = BreakableCUDAGraphCapture()
with capture:
query_fp8 = MiniMaxM3SparseAttention._allocate_query_fp8(attention, qkv)
query_fp8 = MiniMaxM3SparseAttention._allocate_query_fp8(attention, qkv) # type: ignore[arg-type]
if query_fp8 is not None:
query_fp8.zero_()
run_attention(query_fp8)
@@ -783,6 +783,9 @@ def test_aiter_fa_shared_kv_matches_reference(
32 * BLOCK_SIZE, num_kv_heads, head_size, dtype=torch.bfloat16
)
value = torch.randn_like(key)
from vllm.v1.attention.backends.rocm_aiter_fa import AiterFlashAttentionImpl
assert isinstance(target.impl, AiterFlashAttentionImpl)
target.impl.do_kv_cache_update(
target,
key,
@@ -82,6 +82,7 @@ def _run_verify_block():
)
from vllm.config import SpeculativeConfig
from vllm.config.vllm import set_current_vllm_config
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAImpl
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.kv_cache_interface import MLAAttentionSpec
from vllm.v1.worker.workspace import init_workspace_manager
@@ -106,7 +107,7 @@ def _run_verify_block():
vllm_config.speculative_config = SpeculativeConfig(
method="ngram", num_speculative_tokens=QLEN - 1
)
vllm_config.model_config.get_num_attention_heads = types.MethodType(
vllm_config.model_config.get_num_attention_heads = types.MethodType( # type: ignore[method-assign]
lambda self, parallel_config, arch_config=None: NUM_QUERY_HEADS,
vllm_config.model_config,
)
@@ -202,7 +203,8 @@ def _run_verify_block():
lambda: spy,
),
):
impl.forward_mqa((q_nope, q_pe), kv_cache, metadata, layer=None)
assert isinstance(impl, AiterMLAImpl)
impl.forward_mqa((q_nope, q_pe), kv_cache, metadata, layer=None) # type: ignore[arg-type] # Spy decode path does not read layer.
return metadata, captured
@@ -121,7 +121,7 @@ def _build_prefill_metadata(
builder = object.__new__(AiterMLAMetadataBuilder)
builder.num_heads = num_heads
builder.mla_dims = SimpleNamespace(v_head_dim=V_HEAD_DIM)
builder.mla_dims = SimpleNamespace(v_head_dim=V_HEAD_DIM) # type: ignore[assignment]
builder._init_fp8_prefill_ps_buffers(
max_num_reqs=len(seq_lens),
max_prefill_qlen=max_q,
@@ -136,7 +136,7 @@ def _build_prefill_metadata(
prefill = SimpleNamespace(query_start_loc=qo_indptr, max_query_len=max_q)
metadata = SimpleNamespace(prefill=prefill, num_decodes=0)
common = SimpleNamespace(query_start_loc_cpu=qo_indptr_cpu)
builder._build_fp8_prefill_ps_metadata(metadata, common)
builder._build_fp8_prefill_ps_metadata(metadata, common) # type: ignore[arg-type]
return metadata, total_q, builder
@@ -167,7 +167,10 @@ def test_sparse_mla_sink_matches_ragged_reference(
impl.sinks = sinks
output, lse = impl._forward_mla(
SimpleNamespace(_q_scale=q_scale, _k_scale=kv_scale), q, kv, metadata
SimpleNamespace(_q_scale=q_scale, _k_scale=kv_scale),
q,
kv,
metadata, # type: ignore[arg-type]
)
kv_flat = kv_ref[:, 0]
references = []
@@ -241,7 +244,7 @@ def test_sparse_mla_sink_rejects_unsupported_aiter_dtypes(
SimpleNamespace(_q_scale=None, _k_scale=None),
torch.empty(1, 16, Q_HEAD_DIM, dtype=q_dtype),
torch.empty(1, 1, Q_HEAD_DIM, dtype=kv_dtype),
metadata,
metadata, # type: ignore[arg-type]
)
@@ -381,7 +384,7 @@ def test_sparse_mla_sink_matches_dense_attention_with_empty_rows_and_paged_cache
paged_kv_indices=torch.tensor(indices, dtype=torch.int32, device="cuda"),
paged_kv_indptr=lengths.cumsum(0).to(device="cuda", dtype=torch.int32),
)
impl = ROCMAiterMLASparseImpl.__new__(ROCMAiterMLASparseImpl)
impl = ROCMAiterMLASparseImpl.__new__(ROCMAiterMLASparseImpl) # type: ignore[arg-type]
impl.num_heads = num_heads
impl.head_size = head_dim
impl.kv_lora_rank = value_dim
@@ -438,9 +441,9 @@ def test_sparse_mla_backend_resolves_only_contiguous_layer_layouts(monkeypatch,
]
if layout == "BLNHC":
with pytest.raises(ValueError, match="does not satisfy"):
resolve_kv_cache_layout(config, supported)
resolve_kv_cache_layout(config, supported) # type: ignore[arg-type]
else:
assert resolve_kv_cache_layout(config, supported).name == layout
assert resolve_kv_cache_layout(config, supported).name == layout # type: ignore[arg-type]
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@@ -491,7 +494,7 @@ def test_sparse_mla_sink_forward_mqa_preserves_split_query(dtype):
actual, _ = impl.forward_mqa(
(q[..., :V_HEAD_DIM], q[..., V_HEAD_DIM:]),
kv,
metadata,
metadata, # type: ignore[arg-type]
SimpleNamespace(_q_scale=None, _k_scale=None),
)
references = [
@@ -39,10 +39,10 @@ def _make_builder():
topk_tokens = 4
builder.device = torch.device("cpu")
builder.kv_cache_spec = SimpleNamespace(block_size=1)
builder.kv_cache_spec = SimpleNamespace(block_size=1) # type: ignore[assignment]
builder.model_dtype = torch.bfloat16
builder.kv_cache_dtype = "fp8"
builder.mla_dims = SimpleNamespace(kv_lora_rank=512, qk_rope_head_dim=64)
builder.mla_dims = SimpleNamespace(kv_lora_rank=512, qk_rope_head_dim=64) # type: ignore[assignment]
builder.topk_tokens = topk_tokens
builder.req_id_per_token_buffer = torch.zeros(
max_num_batched_tokens, dtype=torch.int32, device="cpu"
@@ -227,7 +227,9 @@ def _decode_metadata(case, rows, ratio, query_lens):
num_decode_tokens=len(rows),
num_prefills=0,
num_prefill_tokens=0,
decode=types.SimpleNamespace(block_table=block_table, seq_lens=lens[:, None]),
decode=types.SimpleNamespace( # type: ignore[arg-type]
block_table=block_table, seq_lens=lens[:, None]
),
decode_row_lens=lens,
decode_block_ends=(lens + CAND_BLOCK - 1) // CAND_BLOCK,
decode_native=native,
@@ -278,7 +280,7 @@ def _prefill_metadata(
num_decode_tokens=0,
num_prefills=len(case.seq_lens),
num_prefill_tokens=len(rows),
prefill=types.SimpleNamespace(chunks=chunks),
prefill=types.SimpleNamespace(chunks=chunks), # type: ignore[arg-type]
prefill_plans=plans,
gather_launches=plan_gather_launches(chunks, plans, gather_rows),
)
@@ -955,7 +955,7 @@ def test_rocm_capture_metadata_sets_adaptive_marker(monkeypatch) -> None:
)
builder = object.__new__(rocm_mod.DeepseekV4ROCMAiterMLASparseMetadataBuilder)
actual = builder.build_for_cudagraph_capture(SimpleNamespace())
actual = builder.build_for_cudagraph_capture(SimpleNamespace()) # type: ignore[arg-type] # Patched base method ignores metadata.
assert actual is metadata
assert actual.for_cudagraph_capture is _on_gfx950()
@@ -1563,7 +1563,7 @@ def test_dsv4_adaptive_mla_swa_metadata_graph_replay(monkeypatch) -> None:
"sliding_window": window_size,
},
)
vllm_config.speculative_config = SimpleNamespace(
vllm_config.speculative_config = SimpleNamespace( # type: ignore[assignment]
num_speculative_tokens=upper_query_len - 1,
parallel_drafting=False,
enable_adaptive_verification=True,
@@ -153,6 +153,7 @@ def test_triton_unified_attn_diffkv_vs_reference(
else:
from vllm.v1.attention.backends.fa_utils import flash_attn_varlen_func
assert fa_version is not None
# FA's TMA path needs aligned singleton strides (num_kv_heads == 1).
fa_k = canonicalize_singleton_dim_strides(key_cache)
fa_v = canonicalize_singleton_dim_strides(value_cache)
+1 -1
View File
@@ -79,7 +79,7 @@ class TestConfigSet:
config_set = ConfigSet("test_kernel")
with pytest.raises(KeyError, match="platform 'h100' not found"):
config_set.get_config("h100", "nonexistent")
config_set.get_config("h100", CaseKey({"batch": 32}))
config_data = {"num_warps": 8, "num_stages": 4}
data = {
@@ -81,7 +81,7 @@ class TestDynamicPerTokenScaledFp8QuantConfigPicker:
assert selected_key == CaseKey({"hidden_size": 2048, "num_tokens": 32})
def test_config_picker_no_configs(self):
config_keys: list[dict] = []
config_keys: list[CaseKey] = []
args = _generate_fake_input(16, 4096)
selected_key = pick_config(args, config_keys)
@@ -127,7 +127,7 @@ class TestFusedQkNormRopeConfigPicker:
)
def test_config_picker_no_configs(self):
config_keys: list[dict] = []
config_keys: list[CaseKey] = []
args = _generate_fake_input(16, 4096, 128)
selected_key = pick_config(args, config_keys)
@@ -38,6 +38,7 @@ from torch._inductor.pattern_matcher import (
)
from torch.fx.experimental.proxy_tensor import make_fx
from vllm.kernels.helion.case_key import CaseKey
from vllm.kernels.helion.config_manager import ConfigManager
from vllm.kernels.helion.register import HelionKernelWrapper
@@ -45,7 +46,7 @@ from vllm.kernels.helion.register import HelionKernelWrapper
@contextlib.contextmanager
def _helion_mock_context():
configs = {
"default": helion.Config(block_sizes=[64], num_warps=2, num_stages=2),
CaseKey.default(): helion.Config(block_sizes=[64], num_warps=2, num_stages=2),
}
mock_config_manager = Mock(spec=ConfigManager)
mock_config_manager.get_platform_configs = Mock(return_value=configs)
@@ -88,7 +89,7 @@ class TestMakeFxHop:
raw_kernel_func=raw_add_scale,
op_name="test_make_fx",
fake_impl=lambda *a, **kw: None,
config_picker=lambda args, keys: "default",
config_picker=lambda args, keys: CaseKey.default(),
)
def fn(x, y):
@@ -145,7 +146,7 @@ class TestMakeFxHop:
raw_kernel_func=raw_silu_mul,
op_name="test_pm_silu_mul",
fake_impl=lambda *a, **kw: None,
config_picker=lambda args, keys: "default",
config_picker=lambda args, keys: CaseKey.default(),
)
def pattern(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
@@ -107,7 +107,7 @@ class TestPerTokenGroupFp8QuantConfigPicker:
)
def test_config_picker_no_configs(self):
config_keys: list[dict] = []
config_keys: list[CaseKey] = []
args = _generate_fake_input(16, 4096, 128)
selected_key = pick_config(args, config_keys)
+1
View File
@@ -397,6 +397,7 @@ class TestHelionKernelWrapper:
)
assert wrapper._disabled is True
assert wrapper._disabled_reason is not None
assert "No configs available" in wrapper._disabled_reason
def test_disabled_wrapper_raises_on_call(self, sample_kernel):
@@ -89,7 +89,7 @@ class TestRmsNormDynamicPerTokenQuantConfigPicker:
assert selected_key == CaseKey({"hidden_size": 2048, "num_tokens": 32})
def test_config_picker_no_configs(self):
config_keys: list[dict] = []
config_keys: list[CaseKey] = []
args = _generate_fake_input(16, 4096)
selected_key = pick_config(args, config_keys)
@@ -112,7 +112,7 @@ class TestRmsNormPerBlockQuantConfigPicker:
)
def test_config_picker_no_configs(self):
config_keys: list[dict] = []
config_keys: list[CaseKey] = []
args = _generate_fake_input(16, 4096, 128)
selected_key = pick_config(args, config_keys)
@@ -96,7 +96,7 @@ class TestSiluAndMulPerBlockQuantConfigPicker:
)
def test_config_picker_no_configs(self):
config_keys: list[dict] = []
config_keys: list[CaseKey] = []
args = _generate_fake_input(16, 4096, 128)
selected_key = pick_config(args, config_keys)
+2 -1
View File
@@ -83,7 +83,7 @@ class TestSiluMulFp8ConfigPicker:
assert selected_key == CaseKey({"intermediate": 4096, "numtokens": 256})
def test_config_picker_no_configs(self):
config_keys: list[dict] = []
config_keys: list[CaseKey] = []
input_tensor = torch.randn(32, 4096, dtype=torch.bfloat16, device="cuda")
scale = torch.tensor([0.5], dtype=torch.float32, device="cuda")
@@ -355,6 +355,7 @@ class TestSiluMulFp8Integration:
registered_kernels = get_registered_kernels()
kernel_wrapper = registered_kernels["silu_mul_fp8"]
fake_impl = kernel_wrapper._fake_impl
assert fake_impl is not None
fake_output = fake_impl(input_tensor, scale)
@@ -7,6 +7,15 @@ import pytest
from tests.conftest import HfRunner
from tests.models.utils import EmbedModelInfo, check_embeddings_close, matryoshka_fy
from vllm.entrypoints.pooling.embed.protocol import EmbeddingResponse
def float_embeddings(response: EmbeddingResponse) -> list[list[float]]:
embeddings = []
for d in response.data:
assert isinstance(d.embedding, list)
embeddings.append(d.embedding)
return embeddings
def run_embedding_correctness_test(
+9 -9
View File
@@ -53,12 +53,10 @@ def test_video_media_io_kwargs(monkeypatch: pytest.MonkeyPatch):
imageio = ImageMediaIO()
# Verify that different args pass/fail assertions as expected.
videoio = VideoMediaIO(imageio, **{"num_frames": 10, "fps": 1.0})
videoio = VideoMediaIO(imageio, num_frames=10, fps=1.0)
_ = videoio.load_bytes(b"test")
videoio = VideoMediaIO(
imageio, **{"num_frames": 10, "fps": 1.0, "not_used": "not_used"}
)
videoio = VideoMediaIO(imageio, num_frames=10, fps=1.0, not_used="not_used")
_ = videoio.load_bytes(b"test")
with pytest.raises(AssertionError, match="bad num_frames"):
@@ -66,11 +64,11 @@ def test_video_media_io_kwargs(monkeypatch: pytest.MonkeyPatch):
_ = videoio.load_bytes(b"test")
with pytest.raises(AssertionError, match="bad num_frames"):
videoio = VideoMediaIO(imageio, **{"num_frames": 9, "fps": 1.0})
videoio = VideoMediaIO(imageio, num_frames=9, fps=1.0)
_ = videoio.load_bytes(b"test")
with pytest.raises(AssertionError, match="bad fps"):
videoio = VideoMediaIO(imageio, **{"num_frames": 10, "fps": 2.0})
videoio = VideoMediaIO(imageio, num_frames=10, fps=2.0)
_ = videoio.load_bytes(b"test")
@@ -86,14 +84,14 @@ def test_opencv_video_io_colorspace(tmp_path, is_color: bool, fourcc: str, ext:
image = Image.open(image_path)
if not is_color:
image_path = f"{tmp_path}/test_grayscale_image.png"
image_path = Path(tmp_path) / "test_grayscale_image.png"
image = image.convert("L")
image.save(image_path)
# Convert to gray RGB for comparison
image = image.convert("RGB")
video_path = f"{tmp_path}/test_RGB_video.{ext}"
create_video_from_image(
image_path,
str(image_path),
video_path,
num_frames=2,
is_color=is_color,
@@ -385,7 +383,9 @@ def test_pynvvideocodec_unrelated_error_propagates(
with pytest.raises(RuntimeError) as exc_info:
PyNvVideoCodecVideoBackendMixin.decode_frames_pynvvideocodec(
None, b"video", None
None,
b"video",
None, # type: ignore[arg-type] # Fails before target metadata is used.
)
assert exc_info.value is original_error
+13 -1
View File
@@ -179,7 +179,7 @@ def test_audio_resampler_invalid_method(dummy_audio):
# Validated eagerly so a bad method fails at construction, not on the
# first audio request.
with pytest.raises(ValueError, match="Invalid resampling method"):
AudioResampler(target_sr=22050, method="invalid")
AudioResampler(target_sr=22050, method="invalid") # type: ignore[arg-type]
def test_audio_resampler_no_target_sr(dummy_audio):
@@ -378,6 +378,7 @@ class TestMultiModalDataParserChannelNormalization:
result = parser._parse_audio_data((stereo_audio, 16000))
# Check that result is mono (1D)
assert result is not None
audio_item = result.get(0)
assert audio_item.ndim == 1, f"Expected 1D mono audio, got {audio_item.ndim}D"
assert audio_item.shape == (3,), f"Expected shape (3,), got {audio_item.shape}"
@@ -404,6 +405,7 @@ class TestMultiModalDataParserChannelNormalization:
result = parser._parse_audio_data((stereo_audio, 16000))
# Check that result preserves original shape (after resampling)
assert result is not None
audio_item = result.get(0)
# When target_channels=None, stereo audio should be preserved
assert audio_item.ndim == 2, f"Expected 2D stereo audio, got {audio_item.ndim}D"
@@ -425,6 +427,7 @@ class TestMultiModalDataParserChannelNormalization:
result = parser._parse_audio_data((mono_audio, 16000))
# Check that result is still mono (1D)
assert result is not None
audio_item = result.get(0)
assert audio_item.ndim == 1
assert audio_item.shape == (16000,)
@@ -446,6 +449,7 @@ class TestMultiModalDataParserChannelNormalization:
result = parser._parse_audio_data((surround_audio, 16000))
# Check that result is stereo (2 channels)
assert result is not None
audio_item = result.get(0)
assert audio_item.ndim == 2
assert audio_item.shape[0] == 2 # 2 channels
@@ -483,6 +487,7 @@ class TestAudioPipelineE2E:
# Process audio through the parser
result = parser._parse_audio_data((stereo_torchaudio, 16000))
assert result is not None
audio_output = result.get(0)
# Verify output is mono 1D
@@ -512,6 +517,7 @@ class TestAudioPipelineE2E:
# Process audio through the parser
result = parser._parse_audio_data((stereo_soundfile, 16000))
assert result is not None
audio_output = result.get(0)
# Verify output is mono 1D
@@ -537,6 +543,7 @@ class TestAudioPipelineE2E:
# Process audio through the parser
result = parser._parse_audio_data((mono_pyav, 16000))
assert result is not None
audio_output = result.get(0)
# Verify output is still mono 1D
@@ -572,6 +579,7 @@ class TestAudioPipelineE2E:
# Process audio through the parser
result = parser._parse_audio_data((surround_audio, 16000))
assert result is not None
audio_output = result.get(0)
# Verify output is mono 1D
@@ -603,6 +611,7 @@ class TestAudioPipelineE2E:
# Process audio through the parser
# Note: Parser expects numpy, so we convert first (simulating real usage)
result = parser._parse_audio_data((stereo_torch.numpy(), 16000))
assert result is not None
audio_output = result.get(0)
# Verify output is mono 1D numpy array
@@ -630,6 +639,7 @@ class TestAudioPipelineE2E:
# Process audio through the parser
result = parser._parse_audio_data((stereo_audio, 16000))
assert result is not None
audio_output = result.get(0)
# Verify output preserves stereo (2D)
@@ -654,6 +664,7 @@ class TestAudioPipelineE2E:
# Process audio through the parser
result = parser._parse_audio_data((stereo_48k, 48000))
assert result is not None
audio_output = result.get(0)
# Verify output is mono 1D at target sample rate
@@ -677,6 +688,7 @@ class TestAudioPipelineE2E:
)
result = parser._parse_audio_data((short_stereo, 16000))
assert result is not None
audio_output = result.get(0)
# Should still produce mono output
+13 -6
View File
@@ -11,7 +11,6 @@ import torch
from vllm.config import ModelConfig, ParallelConfig, VllmConfig
from vllm.config.multimodal import MultiModalConfig
from vllm.multimodal.cache import (
BaseMultiModalProcessorCache,
BaseMultiModalReceiverCache,
LruKeyReplicatedReceiverCache,
LruKeyReplicatedSenderCache,
@@ -145,6 +144,8 @@ def _compare_caches(
cache_1_p0 = processor_cache_from_config(config_1)
cache_1_p1 = engine_receiver_cache_from_config(config_1)
assert config_0.model_config.multimodal_config is not None
assert config_1.model_config.multimodal_config is not None
cache_size_gb = max(
config_0.model_config.multimodal_config.mm_processor_cache_gb,
config_1.model_config.multimodal_config.mm_processor_cache_gb,
@@ -280,6 +281,7 @@ def test_oversized_item_is_served_uncached():
p0 = LruKeyReplicatedSenderCache(model_config) # type: ignore[arg-type]
p1 = LruKeyReplicatedReceiverCache(model_config) # type: ignore[arg-type]
assert item is not None
assert p0.get_and_update_item((item, []), "big")[0] is item
assert not p0.is_cached_item("big")
assert p1.get_and_update_item(item, "big") is item
@@ -318,6 +320,7 @@ def test_mm_cache_miss_raises_and_recovers():
assert p0.is_cached_item(mm_hash)
# On the next request P0 short-circuits to data=None -- the drift bug when
# P1 lacks the item.
assert item is not None
hit = p0.get_and_update_item((item, []), mm_hash)
assert hit[0] is None
@@ -326,6 +329,7 @@ def test_mm_cache_miss_raises_and_recovers():
p0.invalidate(mm_hash)
assert not p0.is_cached_item(mm_hash)
assert item is not None
resent = p0.get_and_update_item((item, []), mm_hash)
assert resent[0] is item # MISS again -> data is resent
assert p1.get_and_update_item(item, mm_hash) == item # P1 now caches it
@@ -459,6 +463,7 @@ def test_shm_receiver_handles_prefix_covered_items(monkeypatch):
address_item, _ = p0.get_and_update_item((item, []), mm_hash)
first = _feature(mm_hash, address_item)
p1.get_and_update_features([first])
assert first.data is not None
assert torch.equal(first.data["dummy"].data, item["dummy"].data)
# Request 2 (identical, fully prefix-covered): the sender hit takes
@@ -476,6 +481,7 @@ def test_shm_receiver_handles_prefix_covered_items(monkeypatch):
assert covered_uncached.data is None
# The address item is resolved to the cached payload.
assert covered_cached.data is not None
assert torch.equal(covered_cached.data["dummy"].data, item["dummy"].data)
# The hit's writer references were acknowledged by the worker, so the
@@ -488,7 +494,7 @@ def test_shm_receiver_handles_prefix_covered_items(monkeypatch):
def _run_test_cache_eviction_lru(
p0_cache: BaseMultiModalProcessorCache,
p0_cache: LruKeyReplicatedSenderCache,
p1_cache: BaseMultiModalReceiverCache,
base_item_size: int,
):
@@ -603,7 +609,7 @@ def test_cache_eviction_lru_cache():
# image_B is protected from eviction then image_i cannot be added.
# This proving normal eviction and reuse behavior.
def _run_test_cache_eviction_shm(
p0_cache: BaseMultiModalProcessorCache,
p0_cache: ShmObjectStoreSenderCache,
p1_cache: BaseMultiModalReceiverCache,
base_item_size: int,
):
@@ -860,9 +866,9 @@ async def test_release_kv_cache_resends_mm_payload(use_async, release_error):
async def call_release():
if use_async:
await AsyncLLM.release_kv_cache_memory(engine)
await AsyncLLM.release_kv_cache_memory(engine) # type: ignore[arg-type]
else:
LLMEngine.release_kv_cache_memory(engine)
LLMEngine.release_kv_cache_memory(engine) # type: ignore[arg-type]
if release_error:
with pytest.raises(RuntimeError, match=release_error):
@@ -894,9 +900,10 @@ def test_sleep_wake_preserves_mm_cache_consistency():
"""Regression for vllm-project/vllm#42995."""
from vllm import LLM, SamplingParams
from vllm.assets.image import ImageAsset
from vllm.inputs import TextPrompt
image = ImageAsset("stop_sign").pil_image
prompt = {
prompt: TextPrompt = {
"prompt": _SLEEP_VISION_PROMPT,
"multi_modal_data": {"image": image},
}
+2
View File
@@ -131,6 +131,7 @@ def test_hash_collision_video_num_frames():
}
video = MediaWithBytes((frames, metadata), source)
items = MultiModalDataParser()._parse_video_data([video])
assert items is not None
return items.get_all_items_for_hash()[0]
hasher = MultiModalHasher
@@ -155,6 +156,7 @@ def test_hash_video_tensor_frames():
}
video = MediaWithBytes((frames, metadata), source)
items = MultiModalDataParser()._parse_video_data([video])
assert items is not None
return items.get_all_items_for_hash()[0]
np_frames = np.zeros((2, 8, 8, 3), dtype=np.uint8)
+22 -13
View File
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import time
from collections.abc import Sequence
from contextlib import nullcontext
from copy import deepcopy
from types import SimpleNamespace
@@ -868,7 +869,7 @@ def test_limit_mm_per_prompt_apply(model_id, num_images, limit, is_valid):
],
)
def test_budget_caps_prevent_dummy_input_validation_failure(
model_id, user_limit, supported_limit
model_id, user_limit, supported_limit, monkeypatch
):
limit_mm_per_prompt = {"image": user_limit}
@@ -878,7 +879,9 @@ def test_budget_caps_prevent_dummy_input_validation_failure(
)
processor = MULTIMODAL_REGISTRY.create_processor(model_config)
processor.info.get_supported_mm_limits = lambda: {"image": supported_limit}
monkeypatch.setattr(
processor.info, "get_supported_mm_limits", lambda: {"image": supported_limit}
)
# This is what budget.py uses to derive mm_counts
allowed = processor.info.allowed_mm_limits
@@ -936,7 +939,7 @@ def test_hf_processor_init_kwargs(
)
processor = ctx.get_hf_processor(
DummyProcessor, # type: ignore[arg-type]
DummyProcessor,
**inference_kwargs,
)
assert processor.a == expected_kwargs["a"]
@@ -967,7 +970,7 @@ def test_hf_processor_call_kwargs(
tokenizer=None,
)
processor = ctx.get_hf_processor(DummyProcessor) # type: ignore[arg-type]
processor = ctx.get_hf_processor(DummyProcessor)
result = ctx.call_hf_processor(processor, {}, inference_kwargs)
assert result == expected_kwargs
@@ -1278,7 +1281,7 @@ def test_get_supported_mm_processor_kwargs_uses_supported_modalities(
processor: SimpleNamespace,
expected: dict[str, set[str]],
) -> None:
info = BaseProcessingInfo(SimpleNamespace())
info = BaseProcessingInfo(SimpleNamespace()) # type: ignore[arg-type]
info.__dict__["supported_mm_limits"] = supported_mm_limits
monkeypatch.setattr(info, "get_hf_processor", lambda **_: processor)
@@ -1288,7 +1291,7 @@ def test_get_supported_mm_processor_kwargs_uses_supported_modalities(
def test_supported_mm_processor_kwargs_is_cached(
monkeypatch: pytest.MonkeyPatch,
) -> None:
info = BaseProcessingInfo(SimpleNamespace())
info = BaseProcessingInfo(SimpleNamespace()) # type: ignore[arg-type]
expected = {"images_kwargs": {"size"}}
calls = 0
@@ -1541,7 +1544,10 @@ def test_processor_inputs_hashes_scope_kwargs_by_modality():
"video": [np.zeros((2, 8, 8, 3), dtype=np.uint8)],
}
)
mm_uuid_items = {"image": ["image-uuid"], "video": ["video-uuid"]}
mm_uuid_items: dict[str, Sequence[str | None]] = {
"image": ["image-uuid"],
"video": ["video-uuid"],
}
def get_hashes(video_frames: int, image_size: int, video_size: int):
return ProcessorInputs(
@@ -1621,7 +1627,7 @@ def test_processor_inputs_hashes_distinguish_kwargs_shapes(left, right):
[(None, 491520, 491520), (True, 491520, 8192), (True, 128, 128), (False, 128, 128)],
)
def test_dummy_inputs_scheduler_budget(
chunked_prefill, max_model_len, expected_seq_len
chunked_prefill, max_model_len, expected_seq_len, monkeypatch
):
ctx = build_model_context(
"llava-hf/llava-v1.6-mistral-7b-hf",
@@ -1634,11 +1640,9 @@ def test_dummy_inputs_scheduler_budget(
ctx.model_config,
tokenizer=ctx.tokenizer,
)
processor.apply = lambda *args, **kwargs: {"prompt_token_ids": [7]}
kwargs = {}
scheduler_config = None
if chunked_prefill is not None:
kwargs["scheduler_config"] = SchedulerConfig(
scheduler_config = SchedulerConfig(
max_model_len=max_model_len,
is_encoder_decoder=False,
max_num_batched_tokens=8192,
@@ -1646,5 +1650,10 @@ def test_dummy_inputs_scheduler_budget(
enable_chunked_prefill=chunked_prefill,
)
result = processor.get_dummy_mm_inputs({"image": 1}, **kwargs)
monkeypatch.setattr(
processor, "apply", lambda *args, **kwargs: {"prompt_token_ids": [7]}
)
result = processor.get_dummy_mm_inputs(
{"image": 1}, scheduler_config=scheduler_config
)
assert len(result["prompt_token_ids"]) == expected_seq_len
+1 -1
View File
@@ -24,4 +24,4 @@ def test_create_processor_error_uses_served_model_name():
ValueError,
match="friendly-model-name is not a multimodal model",
):
MULTIMODAL_REGISTRY.create_processor(model_config)
MULTIMODAL_REGISTRY.create_processor(model_config) # type: ignore[arg-type]
+26 -17
View File
@@ -7,6 +7,7 @@ import sys
import threading
from contextlib import ExitStack, contextmanager
from pathlib import Path
from typing import Any
import numpy as np
import numpy.typing as npt
@@ -83,24 +84,28 @@ def _fresh_decoder_pool():
@VIDEO_LOADER_REGISTRY.register("test_video_loader_1")
class TestVideoLoader1(VideoLoader):
@classmethod
def load_bytes(cls, data: bytes, num_frames: int = -1) -> npt.NDArray:
return FAKE_OUTPUT_1
def load_bytes(
cls, data: bytes, num_frames: int = -1, **kwargs
) -> tuple[npt.NDArray, dict[str, Any]]:
return FAKE_OUTPUT_1, {}
@VIDEO_LOADER_REGISTRY.register("test_video_loader_2")
class TestVideoLoader2(VideoLoader):
@classmethod
def load_bytes(cls, data: bytes, num_frames: int = -1) -> npt.NDArray:
return FAKE_OUTPUT_2
def load_bytes(
cls, data: bytes, num_frames: int = -1, **kwargs
) -> tuple[npt.NDArray, dict[str, Any]]:
return FAKE_OUTPUT_2, {}
def test_video_loader_registry():
custom_loader_1 = VIDEO_LOADER_REGISTRY.load("test_video_loader_1")
output_1 = custom_loader_1.load_bytes(b"test")
output_1, _ = custom_loader_1.load_bytes(b"test")
np.testing.assert_array_equal(output_1, FAKE_OUTPUT_1)
custom_loader_2 = VIDEO_LOADER_REGISTRY.load("test_video_loader_2")
output_2 = custom_loader_2.load_bytes(b"test")
output_2, _ = custom_loader_2.load_bytes(b"test")
np.testing.assert_array_equal(output_2, FAKE_OUTPUT_2)
@@ -488,7 +493,7 @@ def test_pynvvideocodec_failed_rebuild_invalidates_decoder_slot():
old_decoder = FakeDecoder()
slot = PyNvVideoCodecDecoderSlot(FakeStream())
slot.decoder = old_decoder
slot.decoder = old_decoder # type: ignore[assignment] # Fake retained GPU decoder.
slot.source_path = "valid.mp4"
class FakeNvc:
@@ -596,7 +601,9 @@ def test_pynvvideocodec_h200_recovers_after_unsupported_8k():
_pynv_decoder_pool.max_slots = old_max
def test_pynvvideocodec_cross_subclass_shares_single_pool():
def test_pynvvideocodec_cross_subclass_shares_single_pool(
monkeypatch: pytest.MonkeyPatch,
):
"""Regression test for GHSA-j682-9xp5-rrf3.
Multiple subclasses of PyNvVideoCodecVideoBackendMixin must share the
@@ -623,11 +630,12 @@ def test_pynvvideocodec_cross_subclass_shares_single_pool():
with _fresh_decoder_pool() as pool:
pool.max_slots = 2
orig_create = PyNvVideoCodecVideoBackendMixin._create_decoder_slot
PyNvVideoCodecVideoBackendMixin._create_decoder_slot = classmethod(
fake_create_slot
)
try:
with monkeypatch.context() as m:
m.setattr(
PyNvVideoCodecVideoBackendMixin,
"_create_decoder_slot",
classmethod(fake_create_slot),
)
with ExitStack() as stack:
stack.enter_context(MixinSubclassA._borrow_decoder_slot())
stack.enter_context(MixinSubclassB._borrow_decoder_slot())
@@ -652,8 +660,6 @@ def test_pynvvideocodec_cross_subclass_shares_single_pool():
assert create_count == 2
assert len(pool.slots) == 2
finally:
PyNvVideoCodecVideoBackendMixin._create_decoder_slot = orig_create
@pytest.mark.parametrize("hw_decoders", [0, -1, 1.5, True, "2"])
@@ -662,7 +668,7 @@ def test_pynvvideocodec_rejects_invalid_hw_decoders(hw_decoders: object):
VideoBackend.load_bytes(
b"fake video",
backend=PYNVVIDEOCODEC_VIDEO_BACKEND,
hw_decoders=hw_decoders, # type: ignore[arg-type]
hw_decoders=hw_decoders,
)
@@ -799,6 +805,7 @@ def test_video_processor_from_model_repo(
)
backend = get_video_loader_backend_for_processor(video_processor)
assert backend is not None
loader = VIDEO_LOADER_REGISTRY.load(backend)
assert isinstance(loader, expected_loader_cls), (
f"{model_repo!r}: backend={backend!r} loaded "
@@ -1693,7 +1700,9 @@ def test_glm5next_backend_codec_parity(tmp_path, backend):
path = _write_gray_video(tmp_path, total_frames, fps)
# Dense default sampling (gap 5) and a sparse max_frames cap (gap 20).
for max_frames in (None, 6):
kwargs = {} if max_frames is None else {"max_frames": max_frames}
kwargs: dict[str, Any] = (
{} if max_frames is None else {"max_frames": max_frames}
)
expected = glm_sample_frame_indices(
total_frames, float(fps), 12.0, max_frame_count=max_frames
)
+4 -4
View File
@@ -123,8 +123,8 @@ def test_auto_gptq_moe_creates_zero_initialized_expert_biases():
method = object.__new__(AutoGPTQMoEMethod)
method.quant_config = AutoGPTQConfig(4, 128, False, True, False, {}, {})
method.input_dtype = None
method.experts_cls = None
method.moe = SimpleNamespace(w13_num_shards=2)
method.experts_cls = None # type: ignore[assignment] # Weight creation does not select experts.
method.moe = SimpleNamespace(w13_num_shards=2) # type: ignore[assignment]
layer = torch.nn.Module()
method.create_weights(
@@ -170,7 +170,7 @@ def test_routed_experts_loads_per_expert_biases():
("w1", torch.tensor([1.0, 2.0, 3.0, 4.0])),
("w3", torch.tensor([5.0, 6.0, 7.0, 8.0])),
):
assert RoutedExperts.weight_loader(
assert RoutedExperts.weight_loader( # type: ignore[call-overload]
loader,
w13_bias,
loaded,
@@ -180,7 +180,7 @@ def test_routed_experts_loads_per_expert_biases():
return_success=True,
)
assert RoutedExperts.weight_loader(
assert RoutedExperts.weight_loader( # type: ignore[call-overload]
loader,
w2_bias,
torch.tensor([9.0, 10.0, 11.0, 12.0]),
+9 -9
View File
@@ -193,7 +193,7 @@ class DummyFusedMoE:
def make_config(**overrides) -> INCConfig:
kwargs = {
kwargs: dict[str, Any] = {
"weight_bits": 4,
"group_size": 128,
"sym": True,
@@ -208,7 +208,7 @@ def make_config(**overrides) -> INCConfig:
def make_layer_config(**overrides) -> INCLayerConfig:
kwargs = {
kwargs: dict[str, Any] = {
"bits": 4,
"group_size": 128,
"sym": True,
@@ -423,7 +423,7 @@ def test_inc_get_quant_method_unquantized_routed_experts_with_model_prefix(
config = make_config(extra_config={"model.moe_layer": {"bits": 16}})
layer = object.__new__(RoutedExperts)
layer.moe_config = None
layer.moe_config = None # type: ignore[assignment] # No MoE config used by this stub.
method = config.get_quant_method(layer, "moe_layer")
@@ -730,7 +730,7 @@ def test_wna16_cuda_low_bit_moe_routes_to_humming(monkeypatch, bits) -> None:
)
layer = object.__new__(RoutedExperts)
layer.moe_config = DummyMoeConfig()
layer.moe_config = DummyMoeConfig() # type: ignore[assignment]
layer_config = make_layer_config(bits=bits)
method = INCWna16Scheme().get_moe_method(
make_config(), layer, "model.layers.0.mlp", layer_config
@@ -861,7 +861,7 @@ def test_qwen3_30b_a3b_w4a16_autoround_routes_to_gptq_moe(
config = make_qwen3_autoround_config("qwen3_30b_a3b_w4a16")
layer = object.__new__(RoutedExperts)
layer.moe_config = DummyMoeConfig()
layer.moe_config = DummyMoeConfig() # type: ignore[assignment]
method = config.get_quant_method(layer, "model.layers.0.mlp")
@@ -890,7 +890,7 @@ def test_qwen3_30b_a3b_mxfp4_autoround_routes_to_mxfp4_moe(
config = make_qwen3_autoround_config("qwen3_30b_a3b_mxfp4")
layer = object.__new__(RoutedExperts)
layer.moe_config = DummyMoeConfig()
layer.moe_config = DummyMoeConfig() # type: ignore[assignment]
ignored_method = config.get_quant_method(
object.__new__(LinearBase), "model.layers.0.self_attn.q_proj"
@@ -1186,7 +1186,7 @@ def test_wna16_xpu_moe_routes_to_gptq_moe(monkeypatch) -> None:
)
layer = object.__new__(RoutedExperts)
layer.moe_config = DummyMoeConfig()
layer.moe_config = DummyMoeConfig() # type: ignore[assignment]
method = INCWna16Scheme().get_moe_method(
make_config(),
layer,
@@ -1669,7 +1669,7 @@ def test_inc_get_quant_method_unquantized_moe_returns_unquantized(
when extra_config has bits >= 16."""
config = make_config(extra_config={"layer": {"bits": 16}})
layer = object.__new__(RoutedExperts)
layer.moe_config = None # UnquantizedFusedMoEMethod accepts moe_config
layer.moe_config = None # type: ignore[assignment] # No MoE config used by this stub.
class DummyUnquantizedFusedMoEMethod:
def __init__(self, moe_config) -> None:
@@ -2094,7 +2094,7 @@ def _with_w4a8_kernel(monkeypatch) -> None:
def _dispatch(layer_config=None):
return INCWna16Scheme().get_linear_method(
object(), object(), "layer", layer_config or make_layer_config()
make_config(), object(), "layer", layer_config or make_layer_config()
)
+1
View File
@@ -99,6 +99,7 @@ def test_machete_rejects_invalid_config(
)
can_impl, reason = MacheteLinearKernel.can_implement(config)
assert not can_impl
assert reason is not None
assert expected_reason in reason
+27 -14
View File
@@ -25,6 +25,7 @@ from vllm.model_executor.layers.attention.attention import (
set_default_quant_scales,
)
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
from vllm.model_executor.layers.quantization.fp8 import (
Fp8Config,
Fp8LinearMethod,
@@ -79,17 +80,22 @@ def test_deepseek_v41_mxfp8_scale_loading(
for module in (model_module, linear_module):
monkeypatch.setattr(module, "get_tensor_model_parallel_world_size", lambda: 2)
monkeypatch.setattr(module, "get_tensor_model_parallel_rank", lambda: tp_rank)
tp_args = dict(quant_config=quant_config, bias=False)
block = torch.nn.Module()
block.attn = torch.nn.Module()
block.attn.wq_b = ColumnParallelLinear(128, 128, **tp_args)
block.attn.fused_wqa_wkv = MergedColumnParallelLinear(128, [128, 64], **tp_args)
block.attn.wq_b = ColumnParallelLinear(
128, 128, quant_config=quant_config, bias=False
)
block.attn.fused_wqa_wkv = MergedColumnParallelLinear(
128, [128, 64], quant_config=quant_config, bias=False
)
block.ffn = torch.nn.Module()
block.ffn.shared_experts = torch.nn.Module()
block.ffn.shared_experts.gate_up_proj = MergedColumnParallelLinear(
128, [128, 128], **tp_args
128, [128, 128], quant_config=quant_config, bias=False
)
block.ffn.shared_experts.down_proj = RowParallelLinear(
128, 128, quant_config=quant_config, bias=False
)
block.ffn.shared_experts.down_proj = RowParallelLinear(128, 128, **tp_args)
block.ffn.experts = torch.nn.Module()
block.ffn.experts.register_parameter(
"weight_scale", torch.nn.Parameter(torch.empty(3, 4, dtype=torch.uint8), False)
@@ -111,9 +117,11 @@ def test_deepseek_v41_mxfp8_scale_loading(
model.config = SimpleNamespace(num_attention_heads=2)
model.quant_config = quant_config
model.use_sequence_parallel = False
model.get_expert_mapping = lambda: [
("experts.weight_scale", "experts.0.w1.weight_scale", 0, "w1")
]
monkeypatch.setattr(
model,
"get_expert_mapping",
lambda: [("experts.weight_scale", "experts.0.w1.weight_scale", 0, "w1")],
)
checkpoints = [
("attn.wq_b", "attn.wq_b", 128, 128, 0, None),
@@ -211,7 +219,10 @@ def test_deepseek_v41_vl_mapper_routes_linear_scales(
vllm_config = SimpleNamespace(
quant_config=SimpleNamespace(weight_block_size=weight_block_size)
)
resolved = model_module._linear_scale_param_name(vllm_config, expert_dtype)
resolved = model_module._linear_scale_param_name(
vllm_config, # type: ignore[arg-type]
expert_dtype,
)
assert resolved == scale_name
mapper = vl_module._make_deepseek_v4_vl_weights_mapper(expert_dtype, resolved)
@@ -373,7 +384,7 @@ def test_deepgemm_mxfp8_preserves_weight_and_scale_values(
@pytest.mark.parametrize("config_source", ["deepseek", "mxfp8"])
@pytest.mark.parametrize("num_tokens", [1, 7, 128])
def test_mxfp8_bmm_loads_and_projects_grouped_weights(
dist_init, default_vllm_config, prequantized, config_source, num_tokens
dist_init, default_vllm_config, monkeypatch, prequantized, config_source, num_tokens
):
"""BMM metadata set after construction selects grouped weight processing."""
from vllm.model_executor.kernels.linear.mxfp8.deep_gemm import (
@@ -398,10 +409,11 @@ def test_mxfp8_bmm_loads_and_projects_grouped_weights(
pytest.skip("DeepGEMM MXFP8 BMM requires Blackwell")
default_vllm_config.model_config = SimpleNamespace(dtype=torch.bfloat16)
quant_config = DeepseekV4FP8Config(
deepseek_config = DeepseekV4FP8Config(
is_checkpoint_fp8_serialized=True, weight_block_size=[32, 32]
)
quant_config._resolved_expert_dtype = "fp4"
deepseek_config._resolved_expert_dtype = "fp4"
quant_config: QuantizationConfig = deepseek_config
if config_source == "mxfp8":
quant_config = get_quantization_config("mxfp8").from_config(
{"quant_method": "mxfp8"}
@@ -432,7 +444,7 @@ def test_mxfp8_bmm_loads_and_projects_grouped_weights(
model.config = SimpleNamespace(num_attention_heads=2)
model.quant_config = quant_config
model.use_sequence_parallel = False
model.get_expert_mapping = lambda: []
monkeypatch.setattr(model, "get_expert_mapping", lambda: [])
model.load_weights(
[
("layers.0.attn.wo_a.weight", weight),
@@ -443,6 +455,7 @@ def test_mxfp8_bmm_loads_and_projects_grouped_weights(
weight.float().reshape(8, 32, 16, 32)
* torch.exp2(scales.float() - 127)[:, None, :, None]
).reshape(2, 128, 512)
assert isinstance(linear.quant_method, ModelOptLinearMethod)
linear.quant_method.process_weights_after_loading(linear)
linear.quant_method.process_weights_after_loading(linear)
assert isinstance(linear.quant_method.kernel, DeepGemmMxfp8BmmLinearKernel)
@@ -785,7 +798,7 @@ def test_kv_cache_scale_sync_to_host_copies():
set_default_quant_scales(layer, register_buffer=True)
layer.kv_cache_dtype = "fp8"
method = BaseKVCacheMethod(quant_config=None)
method = BaseKVCacheMethod(quant_config=None) # type: ignore[arg-type] # Scale synchronization does not use quant_config.
method.create_weights(layer)
# 0.3 stays != 1.0 even after the fp8_fnuz x2 rescale.
checkpoint_scale = torch.tensor(0.3, dtype=torch.float32)
+2 -1
View File
@@ -8,6 +8,7 @@ These tests run on real hardware — no mocks. Skipped on non-GFX950 platforms.
import pytest
import torch
from vllm.config.kernel import MoEBackend
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
@@ -53,7 +54,7 @@ def disable_rocm_aiter(monkeypatch: pytest.MonkeyPatch):
set_rocm_aiter(monkeypatch, False)
def _make_w4a4_moe_config(moe_backend: str = "auto") -> FusedMoEConfig:
def _make_w4a4_moe_config(moe_backend: MoEBackend = "auto") -> FusedMoEConfig:
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
return FusedMoEConfig(
+2 -1
View File
@@ -10,6 +10,7 @@ CUDA (SM >= 7.5), ROCm, or XPU — not just gfx950.
import pytest
import torch
from vllm.config.kernel import MoEBackend
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
@@ -44,7 +45,7 @@ requires_int8_moe = pytest.mark.skipif(
DEVICE = current_platform.device_type
def _make_int8_moe_config(moe_backend: str = "auto") -> FusedMoEConfig:
def _make_int8_moe_config(moe_backend: MoEBackend = "auto") -> FusedMoEConfig:
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
return FusedMoEConfig(
+2
View File
@@ -15,6 +15,7 @@ from tests.quantization.utils import load_model_without_vllm_runner
from vllm.model_executor.layers.linear import ColumnParallelLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQLinearMethod
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501
CompressedTensorsConfig,
)
@@ -80,6 +81,7 @@ def test_quantized_lm_head_matches_linear(
dtype=torch.bfloat16, head_dtype=None
)
default_vllm_config.kernel_config.linear_backend = "humming"
quant_config: QuantizationConfig
if quant_format is None:
quant_config = ModelOptNvFp4Config(
quant_method=preset,
+1
View File
@@ -843,6 +843,7 @@ def test_modelopt_mixed_precision_builds_w4a16_sibling_config():
}
config = m.ModelOptMixedPrecisionConfig.from_config(hf_quant_config)
assert isinstance(config, m.ModelOptMixedPrecisionConfig)
assert config.nvfp4_config.quant_method == "NVFP4"
assert config.w4a16_nvfp4_config.quant_method == "W4A16_NVFP4"
+7 -5
View File
@@ -218,11 +218,13 @@ def test_gptq_nfirst_weights_pass_through_for_triton():
def test_moe_wna16_setup_forwards_selected_backend(monkeypatch):
method = object.__new__(MoeWNA16Method)
method.experts_cls = object
method.experts_cls = object # type: ignore[assignment] # Opaque mocked-expert sentinel.
method.wna16_backend = WNA16MoEBackend.HUMMING
method.moe = object()
method.moe = object() # type: ignore[assignment] # Opaque mocked-config sentinel.
quant_config = object()
method.get_fused_moe_quant_config = lambda layer: quant_config
monkeypatch.setattr(
method, "get_fused_moe_quant_config", lambda layer: quant_config
)
layer = SimpleNamespace(_expert_routing_tables=lambda: (None, None, None))
captured = {}
kernel = object()
@@ -233,7 +235,7 @@ def test_moe_wna16_setup_forwards_selected_backend(monkeypatch):
monkeypatch.setattr(moe_wna16, "make_wna16_moe_kernel", fake_make_wna16_moe_kernel)
method._setup_kernel(layer)
method._setup_kernel(layer) # type: ignore[arg-type] # Routing-table stub.
assert method.moe_kernel is kernel
assert captured["backend"] == WNA16MoEBackend.HUMMING
@@ -278,7 +280,7 @@ def test_moe_wna16_uses_humming_quant_config(monkeypatch):
),
)
assert method.get_fused_moe_quant_config(layer) is quant_config
assert method.get_fused_moe_quant_config(layer) is quant_config # type: ignore[arg-type] # Identity sentinel.
@pytest.mark.skipif(
+15 -14
View File
@@ -16,6 +16,7 @@ from tests.quantization.utils import (
_test_online_quant_peak_mem_impl,
is_quant_method_supported,
load_model_without_vllm_runner,
quant_config_args,
)
from vllm import _custom_ops as ops
from vllm._aiter_ops import rocm_aiter_ops
@@ -180,7 +181,7 @@ def test_qianfan_online_fp8_keeps_vision_layers_unquantized(
config = OnlineQuantizationConfig(args)
other_config = OnlineQuantizationConfig(args)
model = QianfanOCRForConditionalGeneration.__new__(
QianfanOCRForConditionalGeneration
QianfanOCRForConditionalGeneration # type: ignore[type-abstract]
)
model._patch_quant_config(
SimpleNamespace(vision_config=SimpleNamespace(num_hidden_layers=2)),
@@ -209,9 +210,9 @@ def test_online_nvfp4_reuses_kernel_when_weights_are_reprocessed(
monkeypatch,
) -> None:
method = object.__new__(Nvfp4OnlineMoEMethod)
method.moe = SimpleNamespace(is_act_and_mul=True)
method.nvfp4_backend = object()
method.experts_cls = object
method.moe = SimpleNamespace(is_act_and_mul=True) # type: ignore[assignment]
method.nvfp4_backend = object() # type: ignore[assignment] # Opaque mocked-backend sentinel.
method.experts_cls = object # type: ignore[assignment] # Opaque mocked-expert sentinel.
method.moe_quant_config = None
method.moe_kernel = None
@@ -239,7 +240,7 @@ def test_online_nvfp4_reuses_kernel_when_weights_are_reprocessed(
)
make_kernel = Mock(return_value=kernel)
get_quant_config = Mock(return_value=quant_config)
method.get_fused_moe_quant_config = get_quant_config
monkeypatch.setattr(method, "get_fused_moe_quant_config", get_quant_config)
monkeypatch.setattr(
"vllm.model_executor.layers.quantization.online.nvfp4."
@@ -355,7 +356,7 @@ def test_online_prequantized_compatibility(
checkpoint_config = checkpoint_config_factory()
checkpoint_config.online_quantization_config = OnlineQuantizationConfig(
QuantizationConfigArgs(linear="mxfp8")
quant_config_args(linear="mxfp8")
)
config = checkpoint_config
@@ -408,7 +409,7 @@ def test_online_ignore_keeps_checkpoint_quantization_linear(
quant_config = _fully_quantized_quark_config()
prefix = "model.layers.0.self_attn.o_proj"
quant_config.online_quantization_config = OnlineQuantizationConfig(
QuantizationConfigArgs(linear="mxfp8", ignore=[prefix])
quant_config_args(linear="mxfp8", ignore=[prefix])
)
monkeypatch.setattr(
quant_config.online_quantization_config,
@@ -437,7 +438,7 @@ def test_online_quantization_rejects_prequantized_moe(
prefix = "model.layers.0.mlp.experts"
quant_config = _fully_quantized_quark_config()
quant_config.online_quantization_config = OnlineQuantizationConfig(
QuantizationConfigArgs(linear="mxfp4", moe="mxfp4")
quant_config_args(linear="mxfp4", moe="mxfp4")
)
with pytest.raises(ValueError, match="pre-quantized layer"):
@@ -460,7 +461,7 @@ def test_activation_only_override_applies_to_checkpoint_method(
model_config = ModelConfig(
model=str(tmp_path),
quantization="compressed-tensors",
quantization_config=QuantizationConfigArgs(moe={"activation": "mxfp8"}),
quantization_config=quant_config_args(moe={"activation": "mxfp8"}),
hf_overrides={
"quantization_config": {
"quant_method": "compressed-tensors",
@@ -514,7 +515,7 @@ def test_online_overlay_loads_checkpoint_config_file(tmp_path) -> None:
model_config = ModelConfig(
model=str(tmp_path),
quantization="awq",
quantization_config=QuantizationConfigArgs(linear="mxfp8"),
quantization_config=quant_config_args(linear="mxfp8"),
)
result = weight_utils.get_quant_config(model_config, LoadConfig())
@@ -574,7 +575,7 @@ def test_online_shorthand_selects_checkpoint_or_online_config(
def test_log_online_quantization_for_composable_config(monkeypatch) -> None:
"""Composable configs log their nested online quantization results."""
online_config = OnlineQuantizationConfig(QuantizationConfigArgs(linear="mxfp8"))
online_config = OnlineQuantizationConfig(quant_config_args(linear="mxfp8"))
online_config.quantized_layers = {
"model.layers.0.self_attn.o_proj": ("linear", "mxfp8", None),
"model.layers.1.self_attn.o_proj": ("linear", "mxfp8", None),
@@ -1164,7 +1165,7 @@ def test_online_quantization_records_global_config(
default_vllm_config, dist_init
) -> None:
default_vllm_config.model_config = ModelConfig()
config = OnlineQuantizationConfig(QuantizationConfigArgs(linear="fp8_per_block"))
config = OnlineQuantizationConfig(quant_config_args(linear="fp8_per_block"))
prefix = "model.layers.0.self_attn.o_proj"
layer = ColumnParallelLinear(
input_size=1,
@@ -1218,7 +1219,7 @@ def test_online_quantization_targets_reject_unsupported_layer() -> None:
def test_log_online_quantization(default_vllm_config, monkeypatch) -> None:
config = OnlineQuantizationConfig(QuantizationConfigArgs(linear="fp8_per_tensor"))
config = OnlineQuantizationConfig(quant_config_args(linear="fp8_per_tensor"))
config.quantized_layers = {
"model.layers.0.mlp.down_proj": ("linear", "fp8_per_tensor", None),
"model.layers.1.mlp.down_proj": ("linear", "fp8_per_tensor", None),
@@ -1431,7 +1432,7 @@ def test_online_int8_moe_w2_scale_matches_unsharded(monkeypatch) -> None:
layer.w2_weight = torch.nn.Parameter(w2_in, requires_grad=False)
layer.num_experts = layer.local_num_experts = w13.shape[0]
method = SimpleNamespace(moe=SimpleNamespace(tp_size=moe_tp_size))
Int8OnlineMoEMethod._quantize_weights(method, layer)
Int8OnlineMoEMethod._quantize_weights(method, layer) # type: ignore[arg-type]
return layer.w2_weight, layer.w2_scale
full_weight, full_scale = quantize(w2, 1)
+2 -1
View File
@@ -10,6 +10,7 @@ against the pure-torch reference in `reference_mxfp4.py`.
import pytest
import torch
from vllm.config.kernel import MoEBackend
from vllm.config.model import ModelConfig
from vllm.model_executor.kernels.linear import _POSSIBLE_MXFP4_KERNELS
from vllm.model_executor.kernels.linear.mxfp4.aiter import (
@@ -282,7 +283,7 @@ def test_online_mxfp4_tp_weight_quant_matches_unsharded(tp_size: int):
],
)
def test_online_mxfp4_moe_matches_quark(
moe_backend: str,
moe_backend: MoEBackend,
unpadded_hidden_size: int,
unpadded_intermediate_size: int,
default_vllm_config,
@@ -6,6 +6,7 @@ from unittest.mock import Mock
import pytest
from tests.quantization.utils import quant_config_args, quant_spec
from vllm.config.quantization import (
QUANT_KEY_NAMES,
QuantizationConfigArgs,
@@ -30,7 +31,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
def test_quant_spec_resolves_string_to_quant_key():
spec = QuantSpec(weight="mxfp8", activation="fp8_per_token")
spec = quant_spec(weight="mxfp8", activation="fp8_per_token")
assert spec.weight == kMxfp8Dynamic
assert spec.activation == kFp8DynamicTokenSym
@@ -42,12 +43,12 @@ def test_quant_spec_accepts_quant_key_directly():
def test_quant_spec_string_representation_uses_quantization_name():
assert str(QuantSpec(weight="mxfp4")) == "mxfp4"
assert str(quant_spec(weight="mxfp4")) == "mxfp4"
def test_quant_spec_rejects_unknown_name():
with pytest.raises(ValueError, match="unknown quantization name"):
QuantSpec(weight="not_a_real_format")
quant_spec(weight="not_a_real_format")
# ---- QuantizationConfigArgs string shorthand on linear/moe ----------------
@@ -55,7 +56,7 @@ def test_quant_spec_rejects_unknown_name():
def test_args_linear_string_resolves_via_quant_key_names():
# A bare QUANT_KEY_NAMES entry desugars to QuantSpec(weight=<key>).
args = QuantizationConfigArgs(linear="fp8_per_block_static")
args = quant_config_args(linear="fp8_per_block_static")
assert args.linear == QuantSpec(weight=kFp8Static128BlockSym)
assert args.moe is None
@@ -65,7 +66,7 @@ def test_args_moe_string_resolves_via_online_shorthand():
# (so `linear: "fp8_per_block"` and `moe: "fp8_per_block"` produce the
# same per-layer-kind spec the `--quantization fp8_per_block` shorthand
# would).
args = QuantizationConfigArgs(moe="fp8_per_block")
args = quant_config_args(moe="fp8_per_block")
assert args.moe == QuantSpec(weight=kFp8Static128BlockSym)
@@ -73,11 +74,11 @@ def test_args_string_shorthand_missing_slot_raises():
# int8_per_channel_weight_only sets only `moe`; using it on `linear`
# has no defined spec and should raise rather than silently no-op.
with pytest.raises(ValueError, match="does not define a linear spec"):
QuantizationConfigArgs(linear="int8_per_channel_weight_only")
quant_config_args(linear="int8_per_channel_weight_only")
def test_args_accepts_dict_form():
args = QuantizationConfigArgs(moe={"activation": "mxfp8"})
args = quant_config_args(moe={"activation": "mxfp8"})
assert args.moe == QuantSpec(weight=None, activation=kMxfp8Dynamic)
@@ -91,6 +92,7 @@ def test_targets_reject_non_string_keys():
def test_resolve_shorthand_only_populates_both_slots():
args = resolve_quantization_config("fp8_per_block", None)
assert args is not None
assert args.linear == QuantSpec(weight=kFp8Static128BlockSym)
assert args.moe == QuantSpec(weight=kFp8Static128BlockSym)
@@ -105,6 +107,7 @@ def test_resolve_int8_shorthand_leaves_linear_unset():
# int8_per_channel_weight_only is MoE-only; linear stays None so that
# OnlineQuantizationConfig leaves Linear layers in full precision.
args = resolve_quantization_config("int8_per_channel_weight_only", None)
assert args is not None
assert args.linear is None
assert args.moe == QuantSpec(weight=kInt8StaticChannelSym)
@@ -113,6 +116,7 @@ def test_resolve_quantization_config_only():
# When only `quantization_config` is given (e.g. for an already-quantized
# checkpoint that needs an activation override), it's returned as-is.
args = resolve_quantization_config(None, {"moe": {"activation": "mxfp8"}})
assert args is not None
assert args.linear is None
assert args.moe == QuantSpec(weight=None, activation=kMxfp8Dynamic)
@@ -124,13 +128,14 @@ def test_resolve_merges_explicit_over_shorthand():
"fp8_per_tensor",
{"linear": "fp8_per_block"},
)
assert args is not None
assert args.linear == QuantSpec(weight=kFp8Static128BlockSym)
assert args.moe == QuantSpec(weight=kFp8StaticTensorSym)
def test_resolve_quantization_config_with_checkpoint_quantization():
args = resolve_quantization_config("gptq", {"linear": "fp8_per_block"})
assert args == QuantizationConfigArgs(linear="fp8_per_block")
assert args == quant_config_args(linear="fp8_per_block")
# ---- QUANT_KEY_NAMES coverage --------------------------------------------
@@ -140,15 +145,15 @@ def test_quant_key_names_round_trip():
# Every advertised name should round-trip through QuantSpec without error
# and produce the same QuantKey it maps to.
for name, expected in QUANT_KEY_NAMES.items():
assert QuantSpec(weight=name).weight == expected, name
assert QuantSpec(activation=name).activation == expected, name
assert quant_spec(weight=name).weight == expected, name
assert quant_spec(activation=name).activation == expected, name
def test_static_block_weight_paired_with_dynamic_block_activation():
# The block-FP8 shorthand pair: 128x128 static weights + 1x128 dynamic
# activations. Pinning this so renames in QUANT_KEY_NAMES don't quietly
# rewire the kernel dispatch.
spec = QuantSpec(weight="fp8_per_block_static", activation="fp8_per_block_dynamic")
spec = quant_spec(weight="fp8_per_block_static", activation="fp8_per_block_dynamic")
assert spec.weight == kFp8Static128BlockSym
assert spec.activation == kFp8Dynamic128Sym
+8 -5
View File
@@ -22,7 +22,7 @@ from packaging import version
from tests.quantization.utils import load_model_without_vllm_runner
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.config.cache import CacheConfig
from vllm.config.cache import CacheConfig, CacheDType
from vllm.forward_context import set_forward_context
from vllm.model_executor import parameter
from vllm.model_executor.kernels.linear.scaled_mm.aiter import (
@@ -1081,6 +1081,7 @@ def test_quant_method_dispatch_mxfp8_moe_raises(default_vllm_config):
)
wk, ak, mcls = config.get_quant_method_target("experts", RoutedExperts)
assert mcls is QuarkOCP_MX_MoEMethod
assert wk is not None
# The OCP MX MoE constructor should fail loudly for MXFP8.
fake_moe_config = MagicMock()
with pytest.raises(ValueError, match="MXFP8 experts are not supported"):
@@ -1321,7 +1322,6 @@ def test_quark_fp8_ptpc_exposes_kernel_input_quant_key(monkeypatch):
scheme = QuarkW8A8Fp8.__new__(QuarkW8A8Fp8)
scheme.weight_qscheme = "per_channel"
scheme.is_static_input_scheme = False
scheme.input_qscheme = "per_channel"
scheme.activation_quant_key = kFp8DynamicTokenSym
scheme.weight_quant_key = kFp8StaticChannelSym
scheme.out_dtype = dtype
@@ -1343,7 +1343,7 @@ def test_quark_fp8_ptpc_exposes_kernel_input_quant_key(monkeypatch):
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
def test_quark_fp8_w_per_tensor_a_per_tensor(
kv_cache_dtype: str, monkeypatch, dist_init, workspace_init
kv_cache_dtype: CacheDType, monkeypatch, dist_init, workspace_init
):
model_path = "amd/Llama-3.1-8B-Instruct-FP8-KV-Quark-test"
checkpoint_scales = {}
@@ -2314,7 +2314,7 @@ class TestQuarkInt4Format:
None,
quant_config.quant_config["global_quant_config"]["weight"],
quant_config.pack_method,
moe_config,
moe_config, # type: ignore[arg-type]
)
layer = _FakeLayer(moe_config)
@@ -2325,7 +2325,10 @@ class TestQuarkInt4Format:
torch.zeros(1, 16, 8 // tp_size, dtype=torch.uint8),
requires_grad=False,
)
loader = method.get_weight_loader(layer, weight_loader=None)
loader = method.get_weight_loader(
layer, # type: ignore[arg-type]
weight_loader=None,
)
loader(
param,
raw_zp.clone(),
@@ -75,7 +75,7 @@ class CustomQuantConfig(QuantizationConfig):
def get_name(self) -> QuantizationMethods:
"""Name of the quantization method."""
return "custom_quant"
return "custom_quant" # type: ignore[return-value] # Runtime registration extends built-in names.
def get_supported_act_dtypes(self) -> list[torch.dtype]:
"""List of supported activation dtypes."""
+2
View File
@@ -12,6 +12,7 @@ from vllm.config import set_current_vllm_config
from vllm.forward_context import set_forward_context
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.model_loader import get_model_loader
from vllm.model_executor.model_loader.default_loader import DefaultModelLoader
from vllm.platforms import current_platform
if current_platform.is_rocm():
@@ -126,6 +127,7 @@ def test_online_quant_config_dict_json(vllm_runner, enable_pickle):
def load_weights(model):
model_loader = get_model_loader(load_config)
assert isinstance(model_loader, DefaultModelLoader)
weights_iterator = model_loader.get_all_weights(model_config, model)
model.load_weights(weights_iterator)
@@ -34,7 +34,7 @@ def test_shared_nvfp4_input_scales_have_writable_storage(monkeypatch):
outputs = prepare_nvfp4_moe_layer_for_fi_or_cutlass(
backend=NvFp4MoeBackend.FLASHINFER_CUTLASS,
layer=layer,
layer=layer, # type: ignore[arg-type]
w13=w13,
w13_scale=w13_scale,
w13_scale_2=weight_scale,
+1 -1
View File
@@ -300,7 +300,7 @@ class TestTurboQuantKVCacheSpec:
# The layer builds an unpacked spec; the worker's spec-collection
# loop applies TQ slot packing via the backend's customize_spec hook.
spec = Attention.get_kv_cache_spec(layer, vllm_config)
spec = Attention.get_kv_cache_spec(layer, vllm_config) # type: ignore[arg-type]
assert isinstance(spec, FullAttentionSpec)
assert spec.kv_quant_mode.is_turboquant
assert spec.state_content_bytes is None
+16 -1
View File
@@ -6,18 +6,31 @@ from typing import Any
import regex as re
import torch
from pydantic import TypeAdapter
from vllm.config import (
CompilationConfig,
CompilationMode,
ModelConfig,
VllmConfig,
set_current_vllm_config,
)
from vllm.config.quantization import QuantizationConfigArgs, QuantSpec
from vllm.model_executor.layers.quantization import get_quantization_config
from vllm.model_executor.model_loader.default_loader import DefaultModelLoader
from vllm.platforms import current_platform
def quant_spec(**kwargs: Any) -> QuantSpec:
"""Build a `QuantSpec` from user-facing names such as `"mxfp8"`."""
return TypeAdapter(QuantSpec).validate_python(kwargs)
def quant_config_args(**kwargs: Any) -> QuantizationConfigArgs:
"""Build `QuantizationConfigArgs` from user-facing shorthand values."""
return TypeAdapter(QuantizationConfigArgs).validate_python(kwargs)
def _limit_num_hidden_layers(
model: torch.nn.Module, num_hidden_layers: int | None
) -> None:
@@ -59,7 +72,9 @@ def load_model_without_vllm_runner(
**(model_config_kwargs or {}),
)
vllm_config_args = dict(vllm_config_kwargs or {})
vllm_config_args.setdefault("compilation_config", CompilationConfig(mode=0))
vllm_config_args.setdefault(
"compilation_config", CompilationConfig(mode=CompilationMode.NONE)
)
vllm_config = VllmConfig(model_config=model_config, **vllm_config_args)
hf_overrides = (model_config_kwargs or {}).get("hf_overrides") or {}
num_hidden_layers = hf_overrides.get("num_hidden_layers")
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from transformers import AutoTokenizer
from transformers import AutoTokenizer, PreTrainedTokenizerBase
from tests.reasoning.utils import run_reasoning_extraction
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
@@ -39,8 +39,10 @@ REASONING_MODEL_NAME = "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B"
@pytest.fixture(scope="module")
def test_tokenizer():
tokenizer = AutoTokenizer.from_pretrained(REASONING_MODEL_NAME)
def test_tokenizer() -> PreTrainedTokenizerBase:
tokenizer: PreTrainedTokenizerBase = AutoTokenizer.from_pretrained(
REASONING_MODEL_NAME
)
# Add custom test tokens
test_tokens = ["<test:think>", "</test:think>", "<alt:start>", "<alt:end>"]
existing_tokens = set(tokenizer.get_vocab().keys())
@@ -66,7 +68,7 @@ class TestBaseThinkingReasoningParserInit:
def test_initialization_with_missing_tokenizer(self):
"""Test that initialization fails without tokenizer."""
with pytest.raises(ValueError, match="model tokenizer must be passed"):
TestThinkingReasoningParser(None)
TestThinkingReasoningParser(None) # type: ignore[arg-type]
def test_initialization_with_missing_tokens(self, test_tokenizer):
"""Test that initialization fails when tokens are not in vocabulary."""
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from transformers import AutoTokenizer
from transformers import AutoTokenizer, PreTrainedTokenizerBase
from tests.reasoning.utils import run_reasoning_extraction
from vllm.reasoning import ReasoningParser, ReasoningParserManager
@@ -15,7 +15,7 @@ REASONING_MODEL_NAME = "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B"
@pytest.fixture(scope="module")
def deepseek_r1_qwen_tokenizer():
def deepseek_r1_qwen_tokenizer() -> PreTrainedTokenizerBase:
return AutoTokenizer.from_pretrained(REASONING_MODEL_NAME)
@@ -279,10 +279,10 @@ def test_reasoning(
# Test extract_content
if param_dict["content"] is not None:
content = parser.extract_content_ids(output_ids)
assert content == deepseek_r1_qwen_tokenizer.convert_tokens_to_ids(
content_ids = parser.extract_content_ids(output_ids)
assert content_ids == deepseek_r1_qwen_tokenizer.convert_tokens_to_ids(
deepseek_r1_qwen_tokenizer.tokenize(param_dict["content"])
)
else:
content = parser.extract_content_ids(output)
assert content == []
content_ids = parser.extract_content_ids(output)
assert content_ids == []
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from transformers import AutoTokenizer
from transformers import AutoTokenizer, PreTrainedTokenizerBase
from vllm.entrypoints.generate.base.protocol import DeltaMessage
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
@@ -17,7 +17,7 @@ REASONING_MODEL_NAME = "deepseek-ai/DeepSeek-V3.1"
@pytest.fixture(scope="module")
def tokenizer():
def tokenizer() -> PreTrainedTokenizerBase:
return AutoTokenizer.from_pretrained(REASONING_MODEL_NAME)
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from transformers import AutoTokenizer
from transformers import AutoTokenizer, PreTrainedTokenizerBase
from tests.reasoning.utils import run_reasoning_extraction
from vllm.reasoning import ReasoningParser, ReasoningParserManager
@@ -13,7 +13,7 @@ REASONING_MODEL_NAME = "baidu/ERNIE-4.5-21B-A3B-Thinking"
@pytest.fixture(scope="module")
def ernie45_tokenizer():
def ernie45_tokenizer() -> PreTrainedTokenizerBase:
return AutoTokenizer.from_pretrained(REASONING_MODEL_NAME)
@@ -4,9 +4,11 @@
import copy
import pytest
from transformers import AutoTokenizer
from transformers import AutoTokenizer, PreTrainedTokenizerBase
from tests.reasoning.utils import run_reasoning_extraction
from vllm.parser.engine.adapters import ParserEngineReasoningAdapter
from vllm.parser.glm47_moe import Glm47MoeParser
from vllm.reasoning import ReasoningParser, ReasoningParserManager
parser_name = "glm45"
@@ -17,7 +19,7 @@ REASONING_MODEL_NAME = "zai-org/GLM-4.7"
@pytest.fixture(scope="module")
def glm45_tokenizer():
def glm45_tokenizer() -> PreTrainedTokenizerBase:
return AutoTokenizer.from_pretrained(REASONING_MODEL_NAME)
@@ -261,13 +263,20 @@ def glm53_style_tokenizer(glm45_tokenizer):
return tokenizer
def _glm_engine(parser: ReasoningParser) -> Glm47MoeParser:
assert isinstance(parser, ParserEngineReasoningAdapter)
engine = parser._parser_engine
assert isinstance(engine, Glm47MoeParser)
return engine
@pytest.mark.parametrize(
"disable_kwargs", [{"enable_thinking": False}, {"thinking": False}]
)
def test_glm53_template_forces_reasoning(disable_kwargs: dict, glm53_style_tokenizer):
parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name)
parser = parser_cls(glm53_style_tokenizer, chat_template_kwargs=disable_kwargs)
assert parser._parser_engine.thinking_enabled
assert _glm_engine(parser).thinking_enabled
output = glm53_style_tokenizer.tokenize(GLM53_LEAK["output"])
output_tokens: list[str] = [
@@ -298,7 +307,7 @@ def test_glm47_template_honors_thinking_disable(glm45_tokenizer):
parser = ReasoningParserManager.get_reasoning_parser(parser_name)(
glm45_tokenizer, chat_template_kwargs={"enable_thinking": False}
)
assert not parser._parser_engine.thinking_enabled
assert not _glm_engine(parser).thinking_enabled
output = glm45_tokenizer.tokenize(GLM53_LEAK["output"])
output_tokens: list[str] = [
@@ -1,10 +1,11 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TypedDict
from typing import TypedDict, cast
import pytest
import regex as re
from transformers import PreTrainedTokenizerBase
from tests.reasoning.utils import run_reasoning_extraction
from vllm.entrypoints.openai.chat_completion.protocol import (
@@ -52,8 +53,8 @@ class GraniteThinkingTokenizer:
@pytest.fixture
def tokenizer():
return GraniteThinkingTokenizer()
def tokenizer() -> PreTrainedTokenizerBase:
return cast(PreTrainedTokenizerBase, GraniteThinkingTokenizer())
# ── Basic reasoning extraction (non-streaming + streaming) ───────────
@@ -173,7 +174,7 @@ def tokenizer():
],
)
def test_granite_thinking_reasoning(
tokenizer: GraniteThinkingTokenizer,
tokenizer: PreTrainedTokenizerBase,
streaming: bool,
param_dict: ReasoningCase,
):
@@ -195,7 +196,7 @@ def test_granite_thinking_reasoning(
def test_granite_thinking_no_content_after_end_token(
tokenizer: GraniteThinkingTokenizer,
tokenizer: PreTrainedTokenizerBase,
):
parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name)
parser = parser_cls(tokenizer)
@@ -215,7 +216,7 @@ def test_granite_thinking_no_content_after_end_token(
@pytest.mark.parametrize("streaming", [False, True])
def test_granite_thinking_whitespace_only_content(
tokenizer: GraniteThinkingTokenizer,
tokenizer: PreTrainedTokenizerBase,
streaming: bool,
):
parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name)
@@ -235,7 +236,7 @@ def test_granite_thinking_whitespace_only_content(
def test_granite_thinking_unterminated_think_block(
tokenizer: GraniteThinkingTokenizer,
tokenizer: PreTrainedTokenizerBase,
):
parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name)
parser = parser_cls(tokenizer)
@@ -260,7 +261,7 @@ def test_granite_thinking_unterminated_think_block(
def test_granite_thinking_disabled_moves_into_content(
tokenizer: GraniteThinkingTokenizer,
tokenizer: PreTrainedTokenizerBase,
):
parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name)
parser = parser_cls(tokenizer)
@@ -285,7 +286,7 @@ def test_granite_thinking_disabled_moves_into_content(
def test_granite_thinking_disabled_with_leading_newline(
tokenizer: GraniteThinkingTokenizer,
tokenizer: PreTrainedTokenizerBase,
):
# With enable_thinking=False, model output goes through the swap
# path: content is initially None (all text classified as
@@ -316,7 +317,7 @@ def test_granite_thinking_disabled_with_leading_newline(
def test_granite_thinking_force_nonempty_content_moves_into_content(
tokenizer: GraniteThinkingTokenizer,
tokenizer: PreTrainedTokenizerBase,
):
parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name)
parser = parser_cls(tokenizer)
@@ -338,7 +339,7 @@ def test_granite_thinking_force_nonempty_content_moves_into_content(
def test_granite_thinking_force_nonempty_no_swap_when_newlines_only(
tokenizer: GraniteThinkingTokenizer,
tokenizer: PreTrainedTokenizerBase,
):
# When </think> IS present and content is newlines-only, lstrip
# removes them but the swap should NOT fire — content was present,
@@ -363,7 +364,7 @@ def test_granite_thinking_force_nonempty_no_swap_when_newlines_only(
def test_granite_thinking_force_nonempty_swaps_when_content_absent(
tokenizer: GraniteThinkingTokenizer,
tokenizer: PreTrainedTokenizerBase,
):
# When </think> IS present but content is truly absent (zero
# characters after </think>, e.g. max_tokens cut), swap fires.
@@ -387,7 +388,7 @@ def test_granite_thinking_force_nonempty_swaps_when_content_absent(
def test_granite_thinking_force_nonempty_keeps_real_content(
tokenizer: GraniteThinkingTokenizer,
tokenizer: PreTrainedTokenizerBase,
):
parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name)
parser = parser_cls(tokenizer)
@@ -412,7 +413,7 @@ def test_granite_thinking_force_nonempty_keeps_real_content(
def test_granite_thinking_keeps_truncated_reasoning(
tokenizer: GraniteThinkingTokenizer,
tokenizer: PreTrainedTokenizerBase,
):
parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name)
parser = parser_cls(tokenizer)
@@ -472,7 +473,7 @@ def _run_parse_delta(parser, tokenizer, text, request):
def test_granite_thinking_streaming_enable_thinking_false(
tokenizer: GraniteThinkingTokenizer,
tokenizer: PreTrainedTokenizerBase,
):
# With enable_thinking=False, the parser (constructed without
# kwargs) starts in REASONING state. All text streams as reasoning
@@ -494,7 +495,7 @@ def test_granite_thinking_streaming_enable_thinking_false(
def test_granite_thinking_streaming_strips_leading_newline(
tokenizer: GraniteThinkingTokenizer,
tokenizer: PreTrainedTokenizerBase,
):
request = ChatCompletionRequest(
model="test-model",
@@ -511,7 +512,7 @@ def test_granite_thinking_streaming_strips_leading_newline(
def test_granite_thinking_streaming_promotes_reasoning_to_content(
tokenizer: GraniteThinkingTokenizer,
tokenizer: PreTrainedTokenizerBase,
):
request = ChatCompletionRequest(
model="test-model",
@@ -527,7 +528,7 @@ def test_granite_thinking_streaming_promotes_reasoning_to_content(
def test_granite_thinking_streaming_no_promotion_with_real_content(
tokenizer: GraniteThinkingTokenizer,
tokenizer: PreTrainedTokenizerBase,
):
request = ChatCompletionRequest(
model="test-model",
@@ -548,7 +549,7 @@ def test_granite_thinking_streaming_no_promotion_with_real_content(
def test_granite_thinking_streaming_no_promotion_without_opt_in(
tokenizer: GraniteThinkingTokenizer,
tokenizer: PreTrainedTokenizerBase,
):
request = ChatCompletionRequest(model="test-model", messages=[])
parser = _make_reasoning_parser(tokenizer)
@@ -564,7 +565,7 @@ def test_granite_thinking_streaming_no_promotion_without_opt_in(
@pytest.mark.parametrize("streaming", [False, True])
def test_granite_thinking_empty_think_block(
tokenizer: GraniteThinkingTokenizer,
tokenizer: PreTrainedTokenizerBase,
streaming: bool,
):
# <think></think>\nHello — empty reasoning, content after newline.
@@ -592,7 +593,7 @@ def test_granite_thinking_empty_think_block(
],
)
def test_granite_thinking_streaming_chunking_independent(
tokenizer: GraniteThinkingTokenizer,
tokenizer: PreTrainedTokenizerBase,
text: str,
expected_reasoning: str,
expected_content: str,
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from transformers import AutoTokenizer
from transformers import AutoTokenizer, PreTrainedTokenizerBase
from tests.reasoning.utils import run_reasoning_extraction
from vllm.reasoning import ReasoningParser, ReasoningParserManager
@@ -15,7 +15,7 @@ REASONING_MODEL_NAME = "HCompany/Holo2-4B"
@pytest.fixture(scope="module")
def tokenizer():
def tokenizer() -> PreTrainedTokenizerBase:
return AutoTokenizer.from_pretrained(REASONING_MODEL_NAME)
@@ -3,6 +3,7 @@
import string
from collections.abc import Sequence
from typing import cast
import pytest
@@ -10,6 +11,7 @@ from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionReque
from vllm.parser.parser_manager import ParserManager
from vllm.reasoning import ReasoningParserManager
from vllm.reasoning.k2_horizon_reasoning_parser import K2HorizonReasoningParser
from vllm.tokenizers import TokenizerLike
pytestmark = pytest.mark.skip_global_cleanup
@@ -59,6 +61,10 @@ class K2Tokenizer:
return "".join(inverse_vocab[token_id] for token_id in ids)
def _k2_tokenizer() -> TokenizerLike:
return cast(TokenizerLike, K2Tokenizer())
def _request(**kwargs) -> ChatCompletionRequest:
return ChatCompletionRequest(
model="k2-horizon",
@@ -70,7 +76,7 @@ def _request(**kwargs) -> ChatCompletionRequest:
@pytest.mark.parametrize("effort", ["high", "medium", "low"])
def test_reasoning_effort_selects_tokens(effort: str):
parser = K2HorizonReasoningParser(
K2Tokenizer(),
_k2_tokenizer(),
chat_template_kwargs={"reasoning_effort": effort},
)
@@ -78,7 +84,7 @@ def test_reasoning_effort_selects_tokens(effort: str):
def test_reasoning_effort_precedence_and_default():
tokenizer = K2Tokenizer()
tokenizer = _k2_tokenizer()
request = _request(
reasoning_effort="medium",
chat_template_kwargs={"reasoning_effort": "low"},
@@ -103,13 +109,13 @@ def test_reasoning_effort_precedence_and_default():
def test_invalid_reasoning_effort_raises(effort: str | None):
with pytest.raises(ValueError, match="Unsupported reasoning_effort"):
K2HorizonReasoningParser(
K2Tokenizer(),
_k2_tokenizer(),
chat_template_kwargs={"reasoning_effort": effort},
)
def test_non_streaming_explicit_and_implicit_boundaries():
parser = K2HorizonReasoningParser(K2Tokenizer())
parser = K2HorizonReasoningParser(_k2_tokenizer())
request = _request()
assert parser.extract_reasoning("<ifm|think>plan</ifm|think>answer", request) == (
@@ -126,7 +132,7 @@ def test_non_streaming_explicit_and_implicit_boundaries():
@pytest.mark.parametrize("reasoning", ["plan", " ", "\n", " \n"])
def test_character_split_reasoning_stream(reasoning: str):
parser = K2HorizonReasoningParser(K2Tokenizer())
parser = K2HorizonReasoningParser(_k2_tokenizer())
reasoning_parts: list[str] = []
content_parts: list[str] = []
@@ -150,7 +156,7 @@ def test_character_split_reasoning_stream(reasoning: str):
def test_latest_boundary_controls_reasoning_state():
tokenizer = K2Tokenizer()
tokenizer = _k2_tokenizer()
parser = K2HorizonReasoningParser(tokenizer)
vocab = tokenizer.get_vocab()
@@ -159,7 +165,7 @@ def test_latest_boundary_controls_reasoning_state():
def test_composed_streaming_reasoning_to_tool_handoff():
tokenizer = K2Tokenizer()
tokenizer = _k2_tokenizer()
tools = [
{
"type": "function",
@@ -6,6 +6,7 @@ import pytest
from vllm.entrypoints.generate.base.protocol import DeltaMessage
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.parser.kimi_k2 import KimiK2Parser
from vllm.reasoning.kimi_k2_reasoning_parser import KimiK2ReasoningParser
from vllm.tokenizers import get_tokenizer
@@ -65,9 +66,13 @@ def test_extract_reasoning_tool_section_ends_reasoning(kimi_k2_tokenizer):
def test_streaming_reasoning_then_content(kimi_k2_tokenizer):
"""Token-by-token streaming: reasoning tokens then content after </think>."""
parser = KimiK2ReasoningParser(kimi_k2_tokenizer)
parser_engine = parser._parser_engine
assert isinstance(parser_engine, KimiK2Parser)
think_id = parser._parser_engine._start_token_id
end_think_id = parser._parser_engine._end_token_id
think_id = parser_engine._start_token_id
end_think_id = parser_engine._end_token_id
assert think_id is not None
assert end_think_id is not None
# Use a real token ID from the tokenizer for regular content
regular_id = kimi_k2_tokenizer.encode("hello", add_special_tokens=False)[0]
@@ -123,9 +128,13 @@ def test_streaming_reasoning_then_content(kimi_k2_tokenizer):
def test_streaming_tool_section_ends_reasoning(kimi_k2_tokenizer):
"""<|tool_calls_section_begin|> in delta ends reasoning during streaming."""
parser = KimiK2ReasoningParser(kimi_k2_tokenizer)
parser_engine = parser._parser_engine
assert isinstance(parser_engine, KimiK2Parser)
think_id = parser._parser_engine._start_token_id
tool_begin_id = parser._parser_engine._tool_section_start_token_id
think_id = parser_engine._start_token_id
tool_begin_id = parser_engine._tool_section_start_token_id
assert think_id is not None
assert tool_begin_id is not None
regular_id = kimi_k2_tokenizer.encode("hello", add_special_tokens=False)[0]
# Tool section token arrives — should transition from reasoning to content
@@ -1,6 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import cast
import pytest
from vllm.entrypoints.generate.base.protocol import DeltaMessage
@@ -10,6 +12,7 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
from vllm.parser.kimi_k3 import KimiK3Parser
from vllm.parser.parser_manager import ParserManager
from vllm.reasoning.kimi_k3_reasoning_parser import KimiK3ReasoningParser
from vllm.tokenizers import TokenizerLike
pytestmark = pytest.mark.skip_global_cleanup
@@ -25,10 +28,13 @@ RESPONSE_OPEN_IDS = [ord(ch) for ch in RESPONSE_OPEN]
class DummyTokenizer:
def __len__(self) -> int:
return 1
def get_vocab(self) -> dict[str, int]:
return {}
def encode(self, text: str, add_special_tokens: bool = False) -> list[int]:
def encode(self, text: str, *args, **kwargs) -> list[int]:
if text == THINK_OPEN:
return [1, 2, 3]
if text == THINK_CLOSE:
@@ -36,6 +42,10 @@ class DummyTokenizer:
return [ord(ch) for ch in text]
def _dummy_tokenizer() -> TokenizerLike:
return cast(TokenizerLike, DummyTokenizer())
class ReasoningOnlyParser(KimiK3Parser):
reasoning_parser_cls = KimiK3ReasoningParser
@@ -84,7 +94,7 @@ def test_extract_reasoning_with_generation_prefix_consumed():
def test_delegating_parser_strips_response_wrapper_without_tool_parser():
parser = ReasoningOnlyParser(DummyTokenizer())
parser = ReasoningOnlyParser(_dummy_tokenizer())
request = ChatCompletionRequest(model="test-model", messages=[])
reasoning, content, tool_calls = parser.parse(
@@ -165,7 +175,7 @@ def test_count_reasoning_tokens_is_zero_when_thinking_disabled():
def test_count_reasoning_tokens_through_delegating_parser():
parser = ReasoningOnlyParser(DummyTokenizer())
parser = ReasoningOnlyParser(_dummy_tokenizer())
assert parser.count_reasoning_tokens([*OPEN_IDS, 9, *CLOSE_IDS, 11]) == 1
@@ -253,7 +263,8 @@ def test_thinking_disabled_streams_content():
def test_delegating_parser_thinking_false_streams_response_content():
parser = ReasoningOnlyParser(
DummyTokenizer(), chat_template_kwargs={"thinking": False}
_dummy_tokenizer(),
chat_template_kwargs={"thinking": False},
)
request = ChatCompletionRequest(
model="test-model",
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from transformers import AutoTokenizer
from transformers import AutoTokenizer, PreTrainedTokenizerBase
from tests.reasoning.utils import run_reasoning_extraction
from vllm.reasoning import ReasoningParser, ReasoningParserManager
@@ -15,7 +15,7 @@ REASONING_MODEL_NAME = "MiniMaxAI/MiniMax-M2"
@pytest.fixture(scope="module")
def minimax_m2_tokenizer():
def minimax_m2_tokenizer() -> PreTrainedTokenizerBase:
return AutoTokenizer.from_pretrained(REASONING_MODEL_NAME)
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from transformers import AutoTokenizer
from transformers import AutoTokenizer, PreTrainedTokenizerBase
from tests.reasoning.utils import run_reasoning_extraction
from vllm.reasoning import ReasoningParser, ReasoningParserManager
@@ -15,7 +15,7 @@ REASONING_MODEL_NAME = "MiniMaxAI/MiniMax-M2"
@pytest.fixture(scope="module")
def minimax_m2_tokenizer():
def minimax_m2_tokenizer() -> PreTrainedTokenizerBase:
return AutoTokenizer.from_pretrained(REASONING_MODEL_NAME)
@@ -195,10 +195,10 @@ def test_reasoning(
# Test extract_content
if param_dict["content"] is not None:
content = parser.extract_content_ids(output_ids)
assert content == minimax_m2_tokenizer.convert_tokens_to_ids(
content_ids = parser.extract_content_ids(output_ids)
assert content_ids == minimax_m2_tokenizer.convert_tokens_to_ids(
minimax_m2_tokenizer.tokenize(param_dict["content"])
)
else:
content = parser.extract_content_ids(output)
assert content == []
content_ids = parser.extract_content_ids(output)
assert content_ids == []
@@ -1,10 +1,11 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TypedDict
from typing import TypedDict, cast
import pytest
import regex as re
from transformers import PreTrainedTokenizerBase
from tests.reasoning.utils import run_reasoning_extraction
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
@@ -48,8 +49,8 @@ class FakeNemotronTokenizer:
@pytest.fixture
def tokenizer():
return FakeNemotronTokenizer()
def tokenizer() -> PreTrainedTokenizerBase:
return cast(PreTrainedTokenizerBase, FakeNemotronTokenizer())
@pytest.mark.parametrize(
@@ -94,7 +95,7 @@ def tokenizer():
],
)
def test_nemotron_v3_reasoning(
tokenizer: FakeNemotronTokenizer,
tokenizer: PreTrainedTokenizerBase,
streaming: bool,
param_dict: ReasoningCase,
):
@@ -113,7 +114,7 @@ def test_nemotron_v3_reasoning(
def test_nemotron_v3_without_thinking_moves_into_content(
tokenizer: FakeNemotronTokenizer,
tokenizer: PreTrainedTokenizerBase,
):
parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name)
parser = parser_cls(tokenizer)
@@ -137,7 +138,7 @@ def test_nemotron_v3_without_thinking_moves_into_content(
def test_nemotron_v3_force_nonempty_content_moves_into_content(
tokenizer: FakeNemotronTokenizer,
tokenizer: PreTrainedTokenizerBase,
):
parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name)
parser = parser_cls(tokenizer)
@@ -159,7 +160,7 @@ def test_nemotron_v3_force_nonempty_content_moves_into_content(
def test_nemotron_v3_force_nonempty_keeps_real_content(
tokenizer: FakeNemotronTokenizer,
tokenizer: PreTrainedTokenizerBase,
):
# When real content follows the closing tag nothing is promoted: the
# content after </think> is returned as-is and reasoning stays separate.
@@ -183,7 +184,7 @@ def test_nemotron_v3_force_nonempty_keeps_real_content(
def test_nemotron_v3_with_thinking_keeps_truncated_reasoning(
tokenizer: FakeNemotronTokenizer,
tokenizer: PreTrainedTokenizerBase,
):
parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name)
parser = parser_cls(tokenizer)
@@ -242,7 +243,7 @@ def _run_parse_delta(parser, tokenizer, text, request):
def test_nemotron_v3_streaming_promotes_reasoning_to_content(
tokenizer: FakeNemotronTokenizer,
tokenizer: PreTrainedTokenizerBase,
):
# Model never closes <think>: reasoning streams normally AND is duplicated
# into content on the terminal delta.
@@ -260,7 +261,7 @@ def test_nemotron_v3_streaming_promotes_reasoning_to_content(
def test_nemotron_v3_streaming_no_promotion_with_real_content(
tokenizer: FakeNemotronTokenizer,
tokenizer: PreTrainedTokenizerBase,
):
request = ChatCompletionRequest(
model="test-model",
@@ -279,7 +280,7 @@ def test_nemotron_v3_streaming_no_promotion_with_real_content(
def test_nemotron_v3_streaming_no_promotion_without_opt_in(
tokenizer: FakeNemotronTokenizer,
tokenizer: PreTrainedTokenizerBase,
):
# Without enable_thinking=False / force_nonempty_content the fallback must
# stay disabled: the response stays reasoning-only, content empty.
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from transformers import AutoTokenizer
from transformers import AutoTokenizer, PreTrainedTokenizerBase
from tests.reasoning.utils import (
StreamingReasoningReconstructor,
@@ -24,7 +24,7 @@ REASONING_MODEL_NAMES = [
@pytest.fixture(scope="module", params=REASONING_MODEL_NAMES)
def qwen3_tokenizer(request):
def qwen3_tokenizer(request) -> PreTrainedTokenizerBase:
return AutoTokenizer.from_pretrained(request.param)
@@ -4,7 +4,8 @@
import pytest
from tests.reasoning.utils import run_reasoning_extraction
from vllm.reasoning import ReasoningParser, ReasoningParserManager
from vllm.reasoning import ReasoningParserManager
from vllm.reasoning.step3p5_reasoning_parser import Step3p5ReasoningParser
from vllm.tokenizers import get_tokenizer
parser_name = "step3p5"
@@ -283,9 +284,8 @@ def test_reasoning(
output_tokens: list[str] = [
step3p5_tokenizer.convert_tokens_to_string([token]) for token in output
]
parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser(parser_name)(
step3p5_tokenizer
)
parser = ReasoningParserManager.get_reasoning_parser(parser_name)(step3p5_tokenizer)
assert isinstance(parser, Step3p5ReasoningParser)
reasoning, content = run_reasoning_extraction(
parser, output_tokens, streaming=streaming
@@ -317,7 +317,7 @@ def test_reasoning(
# Test extract_content
if param_dict["content"] is not None:
content = parser.extract_content_ids(output_ids)
content_ids = parser.extract_content_ids(output_ids)
# Fixed expected token ids for specific test cases
test_id = (
request.node.callspec.id if hasattr(request.node, "callspec") else None
@@ -332,7 +332,7 @@ def test_reasoning(
expected_content_ids = step3p5_tokenizer.convert_tokens_to_ids(
step3p5_tokenizer.tokenize(param_dict["content"])
)
assert content == expected_content_ids
assert content_ids == expected_content_ids
else:
content = parser.extract_content_ids(output_ids)
assert content == []
content_ids = parser.extract_content_ids(output_ids)
assert content_ids == []
+7 -2
View File
@@ -1,6 +1,10 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import cast
from transformers import PreTrainedTokenizerBase
from vllm.entrypoints.generate.base.protocol import DeltaMessage
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.reasoning import ReasoningParser
@@ -102,10 +106,11 @@ def run_reasoning_extraction_streaming(
reconstructor = StreamingReasoningReconstructor()
previous_text = ""
previous_tokens: list[int] = []
tokenizer = cast(PreTrainedTokenizerBase, reasoning_parser.model_tokenizer)
for delta in model_deltas:
token_delta = [
reasoning_parser.vocab.get(token)
for token in reasoning_parser.model_tokenizer.tokenize(delta)
reasoning_parser.vocab[token]
for token in tokenizer.tokenize(delta)
if token in reasoning_parser.vocab
]
current_text = previous_text + delta
+54 -71
View File
@@ -3,7 +3,7 @@
import random
from collections.abc import Callable
from typing import NamedTuple, TypeAlias
from typing import NamedTuple, NoReturn, TypeAlias
import numpy as np
import pytest
@@ -13,6 +13,7 @@ from tests.utils import create_new_process_for_each_test
from tests.v1.sample.utils import (
LogitsprocsTestFakes,
create_fake_logits,
create_mock_reasoning_config,
create_penalty_tensor,
create_prompt_tokens_tensor,
fake_apply_logitsprocs,
@@ -103,14 +104,6 @@ class LogitsProcsRequestParams:
return f"MyClass({summ})"
class MockReasoningConfig:
"""Minimal reasoning config for ``ThinkingBudgetStateHolder`` tests."""
reasoning_start_token_ids = [THINK_START_TOKEN_ID]
reasoning_end_token_ids = [THINK_END_TOKEN_ID]
enabled = True
def _generate_fake_sampling_metadata(
num_output_tokens: int,
batch_size: int,
@@ -131,7 +124,9 @@ def _generate_fake_sampling_metadata(
)
vllm_config = VllmConfig()
vllm_config.reasoning_config = MockReasoningConfig()
vllm_config.reasoning_config = create_mock_reasoning_config(
[THINK_START_TOKEN_ID], [THINK_END_TOKEN_ID]
)
logitsprocs = build_logitsprocs(
vllm_config=vllm_config,
@@ -242,7 +237,7 @@ def _raise_error_invalid(
request_params: LogitsProcsRequestParams,
step_idx: int,
err_cls: type[Exception] = ValueError,
) -> None:
) -> NoReturn:
raise err_cls(
f"Validation failed for step={step_idx}, "
f"batch_index={batch_index}, "
@@ -268,6 +263,7 @@ def _logit_bias_validate(
) -> None:
"""Validate logit bias logitproc applied correctly."""
logit_bias = request_params.params.logit_bias
assert logit_bias is not None
logits_old = test_fakes.logits[persistent_batch[batch_index].workload_index].cpu()
logits_new = logits_new[batch_index].cpu()
for token_id in range(VOCAB_SIZE):
@@ -368,9 +364,7 @@ def _min_tokens_validate(
ref_num_out_tokens = len(request_params.out_tokens)
min_reached = ref_num_out_tokens >= MIN_TOKENS_LEN_THRESHOLD
ref_all_stop_token_ids = request_params.params.all_stop_token_ids
mt_lp: MinTokensLogitsProcessor = next(
test_fakes.get_logitsprocs_by_cls(MinTokensLogitsProcessor)
)
mt_lp = next(test_fakes.get_logitsprocs_by_cls(MinTokensLogitsProcessor))
assert isinstance(mt_lp, MinTokensLogitsProcessor)
min_tok = mt_lp.min_toks.get(batch_index, None)
@@ -689,7 +683,7 @@ class LogitsprocTestHelpers(NamedTuple):
gen_request_fxn: Callable | None = None
logitsprocs_test_mapping = {
logitsprocs_test_mapping: dict[LogitprocType, LogitsprocTestHelpers] = {
STR_NO_LOGITPROC: LogitsprocTestHelpers(eval_fxn=_none_validate),
LogitBiasLogitsProcessor: LogitsprocTestHelpers(
gen_request_fxn=_logit_bias_params, eval_fxn=_logit_bias_validate
@@ -706,22 +700,22 @@ logitsprocs_test_mapping = {
}
def _get_test_cases() -> list[list[str]]:
def _get_test_cases() -> list[list[LogitprocType]]:
"""Each test case is a set of logitsprocs."""
logitsprocs_types = list(logitsprocs_test_mapping.keys())
logitsprocs_types: list[LogitprocType] = list(logitsprocs_test_mapping.keys())
# Isolate thinking-budget handling from other processors to avoid cross-talk.
thinking_id: LogitprocType = STR_THINKING_BUDGET
other_processors = [
other_processors: list[LogitprocType] = [
p for p in logitsprocs_types if p != STR_NO_LOGITPROC and p != thinking_id
]
return (
[[STR_NO_LOGITPROC]]
+ [[logitproc_type, STR_NO_LOGITPROC] for logitproc_type in other_processors]
+ [other_processors]
+ [[thinking_id]]
test_cases: list[list[LogitprocType]] = [[STR_NO_LOGITPROC]]
test_cases.extend(
[logitproc_type, STR_NO_LOGITPROC] for logitproc_type in other_processors
)
test_cases.extend([other_processors, [thinking_id]])
return test_cases
def _generate_fake_step_update(
@@ -774,6 +768,7 @@ def _generate_fake_step_update(
for add_req_params in workload_params[wdx : (wdx + num_step_add_replace)]:
# Replace as many removed requests as possible with added requests
add_remove_idx = batch_update_builder.pop_removed()
assert add_remove_idx is not None
batch_update_builder.added.append(
(
add_remove_idx,
@@ -976,13 +971,6 @@ def test_logitsprocs(
step_idx += 1
class MockReasoningNoEndTokens:
"""Reasoning config with no end token ids (disables enforcement in holder)."""
reasoning_start_token_ids = [THINK_START_TOKEN_ID]
reasoning_end_token_ids: list[int] = []
def test_maybe_create_thinking_budget_holder_without_reasoning():
cfg = VllmConfig()
assert cfg.reasoning_config is None
@@ -1000,7 +988,9 @@ def test_maybe_create_thinking_budget_holder_without_reasoning():
def test_thinking_budget_holder_has_tracked_after_sync_add():
vc = VllmConfig()
vc.reasoning_config = MockReasoningConfig()
vc.reasoning_config = create_mock_reasoning_config(
[THINK_START_TOKEN_ID], [THINK_END_TOKEN_ID]
)
h = ThinkingBudgetStateHolder(
vc.reasoning_config,
vc.scheduler_config.max_num_seqs,
@@ -1030,7 +1020,9 @@ def test_thinking_budget_holder_has_tracked_after_sync_add():
def test_thinking_budget_holder_sync_remove_clears_state():
vc = VllmConfig()
vc.reasoning_config = MockReasoningConfig()
vc.reasoning_config = create_mock_reasoning_config(
[THINK_START_TOKEN_ID], [THINK_END_TOKEN_ID]
)
h = ThinkingBudgetStateHolder(
vc.reasoning_config,
vc.scheduler_config.max_num_seqs,
@@ -1060,7 +1052,9 @@ def test_thinking_budget_holder_sync_remove_clears_state():
def test_thinking_budget_holder_sync_add_without_budget_drops_row():
vc = VllmConfig()
vc.reasoning_config = MockReasoningConfig()
vc.reasoning_config = create_mock_reasoning_config(
[THINK_START_TOKEN_ID], [THINK_END_TOKEN_ID]
)
h = ThinkingBudgetStateHolder(
vc.reasoning_config,
vc.scheduler_config.max_num_seqs,
@@ -1081,7 +1075,9 @@ def test_thinking_budget_holder_sync_add_without_budget_drops_row():
def test_thinking_budget_holder_swap_exchanges_state():
vc = VllmConfig()
vc.reasoning_config = MockReasoningConfig()
vc.reasoning_config = create_mock_reasoning_config(
[THINK_START_TOKEN_ID], [THINK_END_TOKEN_ID]
)
h = ThinkingBudgetStateHolder(
vc.reasoning_config,
vc.scheduler_config.max_num_seqs,
@@ -1125,7 +1121,9 @@ def test_thinking_budget_holder_swap_exchanges_state():
def test_thinking_budget_holder_unidirectional_move():
vc = VllmConfig()
vc.reasoning_config = MockReasoningConfig()
vc.reasoning_config = create_mock_reasoning_config(
[THINK_START_TOKEN_ID], [THINK_END_TOKEN_ID]
)
h = ThinkingBudgetStateHolder(
vc.reasoning_config,
vc.scheduler_config.max_num_seqs,
@@ -1163,7 +1161,9 @@ def test_thinking_budget_holder_unidirectional_move():
def test_thinking_budget_holder_update_state_repeat_indices_last_row_wins():
vc = VllmConfig()
vc.reasoning_config = MockReasoningConfig()
vc.reasoning_config = create_mock_reasoning_config(
[THINK_START_TOKEN_ID], [THINK_END_TOKEN_ID]
)
h = ThinkingBudgetStateHolder(
vc.reasoning_config,
vc.scheduler_config.max_num_seqs,
@@ -1197,7 +1197,7 @@ def test_thinking_budget_holder_update_state_repeat_indices_last_row_wins():
def test_thinking_budget_holder_spec_mode_tensor_layout():
h = ThinkingBudgetStateHolder(
MockReasoningConfig(),
create_mock_reasoning_config([THINK_START_TOKEN_ID], [THINK_END_TOKEN_ID]),
8,
2,
torch.device("cpu"),
@@ -1209,7 +1209,7 @@ def test_thinking_budget_holder_spec_mode_tensor_layout():
def test_thinking_budget_holder_empty_end_tokens_disables_row():
vc = VllmConfig()
vc.reasoning_config = MockReasoningNoEndTokens()
vc.reasoning_config = create_mock_reasoning_config([THINK_START_TOKEN_ID], [])
h = ThinkingBudgetStateHolder(
vc.reasoning_config,
vc.scheduler_config.max_num_seqs,
@@ -1253,7 +1253,9 @@ def test_thinking_budget_enforced_without_penalties():
passing an empty list (the pre-fix behavior) prevents budget enforcement.
"""
vc = VllmConfig()
vc.reasoning_config = MockReasoningConfig()
vc.reasoning_config = create_mock_reasoning_config(
[THINK_START_TOKEN_ID], [THINK_END_TOKEN_ID]
)
budget = 3 # allow 3 thinking tokens
h = ThinkingBudgetStateHolder(
@@ -1346,7 +1348,11 @@ def test_thinking_budget_long_thinking_section_end_marker_found_at_correct_index
"""Test thinking budget enforced for a long thinking run,
then a natural end marker."""
h = ThinkingBudgetStateHolder(
MockReasoningConfig(), 8, 0, torch.device("cpu"), False
create_mock_reasoning_config([THINK_START_TOKEN_ID], [THINK_END_TOKEN_ID]),
8,
0,
torch.device("cpu"),
False,
)
h.sync_batch(
BatchUpdate(
@@ -1356,8 +1362,8 @@ def test_thinking_budget_long_thinking_section_end_marker_found_at_correct_index
moved=(),
)
)
start = MockReasoningConfig.reasoning_start_token_ids
end = MockReasoningConfig.reasoning_end_token_ids
start = [THINK_START_TOKEN_ID]
end = [THINK_END_TOKEN_ID]
out: list[int] = list(start)
h.update_state([out], None, None)
@@ -1396,13 +1402,9 @@ class TestThinkingBudgetReentry:
@staticmethod
def _make_holder(end_token_ids: list[int]) -> ThinkingBudgetStateHolder:
class FakeReasoningConfig:
reasoning_start_token_ids = [TestThinkingBudgetReentry.THINK_START]
reasoning_end_token_ids: list[int] = []
enabled = True
cfg = FakeReasoningConfig()
cfg.reasoning_end_token_ids = end_token_ids
cfg = create_mock_reasoning_config(
[TestThinkingBudgetReentry.THINK_START], end_token_ids
)
return ThinkingBudgetStateHolder(
reasoning_config=cfg,
max_num_seqs=8,
@@ -1526,21 +1528,12 @@ class TestThinkingBudgetNaturalEndReentry:
@staticmethod
def _make_holder(end_token_ids):
from dataclasses import dataclass
from vllm.v1.sample.thinking_budget_state import (
ThinkingBudgetStateHolder,
)
@dataclass
class FakeReasoningConfig:
reasoning_start_token_ids: list[int]
reasoning_end_token_ids: list[int]
enabled: bool = True
cfg = FakeReasoningConfig(
reasoning_start_token_ids=[TestThinkingBudgetNaturalEndReentry.THINK_START],
reasoning_end_token_ids=end_token_ids,
cfg = create_mock_reasoning_config(
[TestThinkingBudgetNaturalEndReentry.THINK_START], end_token_ids
)
return ThinkingBudgetStateHolder(
reasoning_config=cfg,
@@ -1830,21 +1823,11 @@ class TestThinkingBudgetNaturalEndReentry:
natural </think> exit, continue_thinking is cleared and Block 2
enforcement works correctly.
"""
from dataclasses import dataclass
from unittest.mock import MagicMock
from vllm.v1.sample.thinking_budget_state import ThinkingBudgetStateHolder
@dataclass
class FakeReasoningConfig:
reasoning_start_token_ids: list[int]
reasoning_end_token_ids: list[int]
enabled: bool = True
cfg = FakeReasoningConfig(
reasoning_start_token_ids=[self.THINK_START],
reasoning_end_token_ids=self.THINK_END_SINGLE,
)
cfg = create_mock_reasoning_config([self.THINK_START], self.THINK_END_SINGLE)
holder = ThinkingBudgetStateHolder(
reasoning_config=cfg,
max_num_seqs=8,
+10 -6
View File
@@ -57,15 +57,20 @@ prompts = [
class DummyLogitsProcessor(LogitsProcessor):
"""Fake logit processor to support unit testing and examples."""
@classmethod
def validate_params(cls, params: SamplingParams):
target_token: int | None = params.extra_args and params.extra_args.get(
"target_token"
@staticmethod
def _get_target_token(params: SamplingParams) -> int | None:
target_token = (
params.extra_args.get(DUMMY_LOGITPROC_ARG) if params.extra_args else None
)
if target_token is not None and not isinstance(target_token, int):
raise VLLMValidationError(
f"target_token value {target_token} {type(target_token)} is not int"
)
return target_token
@classmethod
def validate_params(cls, params: SamplingParams):
cls._get_target_token(params)
def __init__(
self, vllm_config: "VllmConfig", device: torch.device, is_pin_memory: bool
@@ -78,8 +83,7 @@ class DummyLogitsProcessor(LogitsProcessor):
def update_state(self, batch_update: BatchUpdate | None):
def extract_extra_arg(params: SamplingParams) -> int | None:
self.validate_params(params)
return params.extra_args and params.extra_args.get("target_token")
return self._get_target_token(params)
process_dict_updates(
self.req_info,
@@ -341,6 +341,7 @@ def test_batch_allowed_needs_one_shared_set():
assert states.batch_allowed([]) is None
shared = states.batch_allowed([0])
assert shared is not None
assert shared.tolist() == [1, 2, 3]
assert shared.dtype == torch.int64
assert states.batch_allowed([0, 0]) is shared
@@ -34,6 +34,8 @@ def test_stats_match_reference(vocab: int, dtype: torch.dtype):
)
assert torch.equal(argmax, ref_argmax)
torch.testing.assert_close(entropy, ref_entropy, atol=2e-3, rtol=1e-3)
assert probs is not None
assert ref_probs is not None
torch.testing.assert_close(probs.float(), ref_probs.float(), atol=5e-3, rtol=2e-2)
assert probs.dtype == torch.bfloat16
assert probs.shape == logits.shape
@@ -100,6 +102,7 @@ def test_masked_logits_keep_a_finite_entropy():
assert entropy[0].item() < 1e-6 and entropy[1].item() < 1e-6
assert abs(entropy[2].item() - math.log(2.0)) < 1e-4
assert torch.equal(argmax[[0, 1, 3]], torch.tensor([3, 3, 3], device="cuda"))
assert probs is not None
assert probs[0, 3].item() == 1.0 and probs[0].sum().item() == 1.0
_, _, ref_entropy, _ = sample_row_stats_reference(logits, temps, 1, None)
assert torch.isfinite(ref_entropy).all()
+6 -3
View File
@@ -340,7 +340,7 @@ def _make_v2_state(
device=device,
all_token_ids=SimpleNamespace(gpu=all_tokens),
)
state = DryState(_make_vllm_config(), req_states)
state = DryState(_make_vllm_config(), req_states) # type: ignore[arg-type]
if tokenizer is not None:
# Re-resolve the default set now that a tokenizer is installed.
state._tokenizer = tokenizer
@@ -446,7 +446,7 @@ def test_spec_decode_is_refused_at_construction():
)
config = _make_vllm_config(speculative_config=SimpleNamespace())
with pytest.raises(ValueError, match="speculative decoding"):
DryState(config, req_states)
DryState(config, req_states) # type: ignore[arg-type]
def test_v2_draft_expanded_logits_raise():
@@ -917,7 +917,10 @@ def test_loads_through_the_loader_by_fqcn():
total_len=None,
)
procs = loader.build_custom_logits_processors(
_make_vllm_config(), req_states, False, [DRY_FQCN]
_make_vllm_config(),
req_states, # type: ignore[arg-type]
False,
[DRY_FQCN],
)
assert [type(p) for p in procs] == [DryState]
+21 -7
View File
@@ -16,6 +16,7 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
UnquantizedEmbeddingMethod,
VocabParallelEmbedding,
)
@@ -32,10 +33,17 @@ class _FakeLmHead:
self.tp_size = 1
def _fake_lm_head(
weight: torch.Tensor, quantized: bool = False, shard_indices: object | None = None
) -> VocabParallelEmbedding:
# The dtype tests only need the projection fields, not distributed setup.
return _FakeLmHead(weight, quantized, shard_indices) # type: ignore[return-value]
def _build_processor(vocab_size: int) -> LogitsProcessor:
lp = LogitsProcessor(vocab_size)
# The TP gather is orthogonal to the dtype behavior under test.
lp._gather_logits = lambda logits: logits
lp._gather_logits = lambda logits: logits # type: ignore[method-assign]
return lp
@@ -47,8 +55,9 @@ def test_fp32_head_runs_projection_in_fp32(default_vllm_config):
hidden_states = torch.randn(num_tokens, hidden_size, dtype=torch.bfloat16)
weight = torch.randn(vocab_size, hidden_size, dtype=torch.bfloat16)
logits = lp._get_logits(hidden_states, _FakeLmHead(weight), None)
logits = lp._get_logits(hidden_states, _fake_lm_head(weight), None)
assert logits is not None
assert logits.dtype == torch.float32
assert torch.isfinite(logits).all()
expected = torch.nn.functional.linear(hidden_states.float(), weight.float())
@@ -66,8 +75,9 @@ def test_non_fp32_head_dtype_uses_cast_path(default_vllm_config):
hidden_states = torch.randn(4, hidden_size, dtype=torch.bfloat16)
weight = torch.randn(vocab_size, hidden_size, dtype=torch.bfloat16)
logits = lp._get_logits(hidden_states, _FakeLmHead(weight), None)
logits = lp._get_logits(hidden_states, _fake_lm_head(weight), None)
assert logits is not None
assert logits.dtype == torch.float16
expected = torch.nn.functional.linear(hidden_states.half(), weight.half())
torch.testing.assert_close(logits, expected)
@@ -82,7 +92,7 @@ def test_head_dtype_equal_to_model_dtype_uses_quant_method(default_vllm_config):
hidden_states = torch.randn(4, hidden_size, dtype=torch.bfloat16)
weight = torch.randn(vocab_size, hidden_size, dtype=torch.bfloat16)
lm_head = _FakeLmHead(weight)
lm_head = _fake_lm_head(weight)
with mock.patch.object(
lm_head.quant_method,
@@ -94,6 +104,7 @@ def test_head_dtype_equal_to_model_dtype_uses_quant_method(default_vllm_config):
logits = lp._get_logits(hidden_states, lm_head, None)
apply_mock.assert_called_once()
assert logits is not None
assert logits.dtype == torch.bfloat16
@@ -120,9 +131,10 @@ def test_fp32_head_uses_mm_fast_path_on_device(default_vllm_config):
with mock.patch(
"vllm.model_executor.layers.logits_processor.F.linear"
) as linear_mock:
logits = lp._get_logits(hidden_states, _FakeLmHead(weight), None)
logits = lp._get_logits(hidden_states, _fake_lm_head(weight), None)
linear_mock.assert_not_called()
assert logits is not None
assert logits.dtype == torch.float32
expected = torch.nn.functional.linear(hidden_states.float(), weight.float())
torch.testing.assert_close(logits, expected)
@@ -131,7 +143,7 @@ def test_fp32_head_uses_mm_fast_path_on_device(default_vllm_config):
def test_fp32_head_rejects_quantized_lm_head(default_vllm_config):
lp = _build_processor(64)
lp.head_dtype = torch.float32
lm_head = _FakeLmHead(torch.randn(64, 16, dtype=torch.bfloat16), quantized=True)
lm_head = _fake_lm_head(torch.randn(64, 16, dtype=torch.bfloat16), quantized=True)
with pytest.raises(ValueError, match="unquantized"):
lp._get_logits(torch.randn(4, 16, dtype=torch.bfloat16), lm_head, None)
@@ -197,7 +209,7 @@ def test_get_top_tokens_honors_head_dtype(default_vllm_config):
hidden_states = torch.randn(4, hidden_size, dtype=torch.bfloat16)
weight = torch.randn(vocab_size, hidden_size, dtype=torch.bfloat16)
lm_head = _FakeLmHead(
lm_head = _fake_lm_head(
weight,
shard_indices=types.SimpleNamespace(
num_org_vocab_padding=0, org_vocab_start_index=0
@@ -236,7 +248,9 @@ def test_fp32_head_e2e_no_nan():
for output in outputs:
for completion in output.outputs:
assert completion.logprobs is not None
for token_id, position in zip(completion.token_ids, completion.logprobs):
assert position is not None
# The sampled token survived filtering, so its logprob is finite.
assert math.isfinite(position[token_id].logprob)
# No returned logprob is NaN.
+33 -17
View File
@@ -24,6 +24,7 @@ from vllm import SamplingParams
from vllm.config.model import LogprobsMode
from vllm.distributed import cleanup_dist_env_and_memory
from vllm.exceptions import VLLMValidationError
from vllm.logprobs import Logprob
from vllm.platforms import current_platform
from vllm.v1.engine.input_processor import InputProcessor
@@ -145,7 +146,7 @@ def _repeat_logprob_config(
def _run_and_validate(
vllm_model: VllmRunner,
test_prompts: list[str],
vllm_sampling_params: SamplingParams,
vllm_sampling_params: SamplingParams | list[SamplingParams],
hf_logprobs: list[list[torch.Tensor]],
hf_outputs: list[tuple[list[int], str]],
logprob_prompt_logprob_list: BatchLogprobsSpecType,
@@ -163,11 +164,12 @@ def _run_and_validate(
# Extract request-level (prompt)logprobs config
num_top_logprobs, num_top_prompt_logprobs = logprob_prompt_logprob
assert vllm_result.prompt_token_ids is not None
# Test whether sampled token output is consistent between vLLM and HF
# vLLM prompt+completion should match HF output
if temperature == 0.0:
assert (
vllm_result.prompt_token_ids + vllm_result.outputs[0].token_ids
vllm_result.prompt_token_ids + list(vllm_result.outputs[0].token_ids)
== hf_output[0]
)
else:
@@ -191,7 +193,9 @@ def _run_and_validate(
# Confirm that the output token appears among the logprobs
assert token_id in logprobs
token_in_topk = logprobs[token_id].rank <= num_top_logprobs
rank = logprobs[token_id].rank
assert rank is not None
token_in_topk = rank <= num_top_logprobs
# If the output token is not included in the top K
# logprob, it can return 1 more data
@@ -207,8 +211,10 @@ def _run_and_validate(
output_text = vllm_result.outputs[0].text
output_string_from_most_likely_tokens_lst: list[str] = []
for top_logprobs in vllm_result.outputs[0].logprobs:
top_logprob = next(iter(top_logprobs.values()))
for output_logprobs in vllm_result.outputs[0].logprobs:
assert output_logprobs is not None
top_logprob = next(iter(output_logprobs.values()))
assert top_logprob.decoded_token is not None
output_string_from_most_likely_tokens_lst.append(
top_logprob.decoded_token
)
@@ -226,8 +232,9 @@ def _run_and_validate(
# Compare vLLM sample logprobs to HF
vllm_sample_logprobs = vllm_result.outputs[0].logprobs
for i, top_logprobs in enumerate(vllm_sample_logprobs):
for token_id, sample_logprob in top_logprobs.items():
for i, sample_logprobs in enumerate(vllm_sample_logprobs):
assert sample_logprobs is not None
for token_id, sample_logprob in sample_logprobs.items():
if temperature == 0.0 or i == 0:
logprob = sample_logprob.logprob
torch.testing.assert_close(
@@ -271,9 +278,9 @@ def _run_and_validate(
# Confirm that the prompt token appears among the logprobs
assert prompt_token_id in prompt_logprobs
token_in_topk = (
prompt_logprobs[prompt_token_id].rank <= num_top_prompt_logprobs
)
prompt_rank = prompt_logprobs[prompt_token_id].rank
assert prompt_rank is not None
token_in_topk = prompt_rank <= num_top_prompt_logprobs
# If the prompt token is not included in the top K
# logprob, it can return 1 more data
@@ -294,9 +301,10 @@ def _run_and_validate(
# 1:.
vllm_prompt_logprobs = vllm_result.prompt_logprobs[1:]
for i, vllm_prompt_logprob_dict in enumerate(vllm_prompt_logprobs):
for token_id, logprob in vllm_prompt_logprob_dict.items():
assert vllm_prompt_logprob_dict is not None
for token_id, prompt_lp in vllm_prompt_logprob_dict.items():
torch.testing.assert_close(
logprob.logprob,
prompt_lp.logprob,
hf_logprob[0][i][token_id].item(),
atol=2e-2,
rtol=2e-2,
@@ -478,12 +486,12 @@ def test_prompt_logprob_token_ids_require_v2_model_runner():
params = SamplingParams(prompt_logprob_token_ids=[1, 2])
with patch.object(SamplingParams, "verify"):
with pytest.raises(VLLMValidationError, match="V2 model runner"):
InputProcessor._validate_params(processor, params, ("generate",))
InputProcessor._validate_params(processor, params, ("generate",)) # type: ignore[arg-type]
processor.vllm_config.use_v2_model_runner = True
InputProcessor._validate_params(processor, params, ("generate",))
InputProcessor._validate_params(processor, params, ("generate",)) # type: ignore[arg-type]
processor.vllm_config.cache_config.kv_sharing_fast_prefill = True
with pytest.raises(VLLMValidationError, match="fast-prefill"):
InputProcessor._validate_params(processor, params, ("generate",))
InputProcessor._validate_params(processor, params, ("generate",)) # type: ignore[arg-type]
def test_none_logprobs(vllm_model, example_prompts):
@@ -607,7 +615,9 @@ def test_logprobs_mode(logprobs_mode: LogprobsMode):
total_token_with_logprobs = 0
positive_values = 0
for output in results[0].outputs:
assert output.logprobs is not None
for logprobs in output.logprobs:
assert logprobs is not None
for token_id in logprobs:
logprob = logprobs[token_id]
if logprobs_mode in ("raw_logprobs", "processed_logprobs"):
@@ -648,6 +658,7 @@ def test_prompt_logprobs_mode():
)
assert results[0].prompt_logprobs is not None
assert results[0].prompt_logprobs[1] is not None
assert results[0].prompt_token_ids is not None
tok_id = results[0].prompt_token_ids[1]
values[mode] = results[0].prompt_logprobs[1][tok_id].logprob
finally:
@@ -1228,10 +1239,12 @@ def test_spec_decode_logprobs(
[prompt, prompt], [sampling_params, penalty_sampling_params]
)
# Collect logprobs outputs from reference LLM.
ref_logprobs = []
ref_logprobs: list[Logprob] = []
for results in ref_results:
for output in results.outputs:
assert output.logprobs is not None
for logprobs in output.logprobs:
assert logprobs is not None
ref_logprobs.extend(logprobs.values())
del ref_llm
torch.accelerator.empty_cache()
@@ -1249,10 +1262,12 @@ def test_spec_decode_logprobs(
[prompt, prompt], [sampling_params, penalty_sampling_params]
)
# Collect logprobs outputs from spec decode LLM.
spec_logprobs = []
spec_logprobs: list[Logprob] = []
for results in spec_results:
for output in results.outputs:
assert output.logprobs is not None
for logprobs in output.logprobs:
assert logprobs is not None
spec_logprobs.extend(logprobs.values())
del spec_llm
torch.accelerator.empty_cache()
@@ -1326,6 +1341,7 @@ def test_prompt_logprobs_with_chunking_and_preemption():
"Unexpected number of prompt logprob positions"
)
assert sampling_params.prompt_logprobs is not None
# Each position should have the requested number of logprobs
for pos, logprobs_dict in enumerate(prompt_logprobs):
if logprobs_dict is not None: # First token may be None
+1
View File
@@ -333,6 +333,7 @@ def test_sampler_repetition_penalty(
for batch_idx in range(batch_size):
non_penalized_token_id = logits[batch_idx].argmax().item()
penalized_token_id = logits[batch_idx].argmin().item()
assert sampling_metadata.prompt_token_ids is not None
prompt_tokens = sampling_metadata.prompt_token_ids[batch_idx][:].tolist()
output_tokens = sampling_metadata.output_token_ids[batch_idx]
if repetition_penalty > 1.0:

Some files were not shown because too many files have changed in this diff Show More