[MyPy][3/N] Fix MyPy errors in test groups (part 3) (#55939)
Signed-off-by: Taimys <ffy2281693949@gmail.com> Signed-off-by: Luyi Xiao <81631911+luyixiao95@users.noreply.github.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Luyi Xiao <81631911+luyixiao95@users.noreply.github.com> Co-authored-by: Taimys <ffy2281693949@gmail.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Codex
Luyi Xiao
Taimys
Harry Mellor
Claude Opus 5.5
parent
c4fa6f36a7
commit
5225f1bacb
@@ -116,7 +116,7 @@ class TestParameterSweep:
|
||||
|
||||
def test_from_records_list(self):
|
||||
"""Test creating ParameterSweep from a list of records."""
|
||||
records = [
|
||||
records: list[dict[str, object]] = [
|
||||
{"max_tokens": 100, "temperature": 0.7},
|
||||
{"max_tokens": 200, "temperature": 0.9},
|
||||
]
|
||||
@@ -127,7 +127,7 @@ class TestParameterSweep:
|
||||
|
||||
def test_read_from_dict(self):
|
||||
"""Test creating ParameterSweep from a dict format."""
|
||||
data = {
|
||||
data: dict[str, dict[str, object]] = {
|
||||
"experiment1": {"max_tokens": 100, "temperature": 0.7},
|
||||
"experiment2": {"max_tokens": 200, "temperature": 0.9},
|
||||
}
|
||||
@@ -149,7 +149,7 @@ class TestParameterSweep:
|
||||
|
||||
def test_read_json_list_format(self):
|
||||
"""Test reading JSON file with list format."""
|
||||
records = [
|
||||
records: list[dict[str, object]] = [
|
||||
{"max_tokens": 100, "temperature": 0.7},
|
||||
{"max_tokens": 200, "temperature": 0.9},
|
||||
]
|
||||
@@ -168,7 +168,7 @@ class TestParameterSweep:
|
||||
|
||||
def test_read_json_dict_format(self):
|
||||
"""Test reading JSON file with dict format."""
|
||||
data = {
|
||||
data: dict[str, dict[str, object]] = {
|
||||
"experiment1": {"max_tokens": 100, "temperature": 0.7},
|
||||
"experiment2": {"max_tokens": 200, "temperature": 0.9},
|
||||
}
|
||||
@@ -190,7 +190,7 @@ class TestParameterSweep:
|
||||
def test_unique_benchmark_names_validation(self):
|
||||
"""Test that duplicate _benchmark_name values raise an error."""
|
||||
# Test with duplicate names in list format
|
||||
records = [
|
||||
records: list[dict[str, object]] = [
|
||||
{"_benchmark_name": "exp1", "max_tokens": 100},
|
||||
{"_benchmark_name": "exp1", "max_tokens": 200},
|
||||
]
|
||||
@@ -200,7 +200,7 @@ class TestParameterSweep:
|
||||
|
||||
def test_unique_benchmark_names_multiple_duplicates(self):
|
||||
"""Test validation with multiple duplicate names."""
|
||||
records = [
|
||||
records: list[dict[str, object]] = [
|
||||
{"_benchmark_name": "exp1", "max_tokens": 100},
|
||||
{"_benchmark_name": "exp1", "max_tokens": 200},
|
||||
{"_benchmark_name": "exp2", "max_tokens": 300},
|
||||
@@ -212,7 +212,7 @@ class TestParameterSweep:
|
||||
|
||||
def test_no_benchmark_names_allowed(self):
|
||||
"""Test that records without _benchmark_name are allowed."""
|
||||
records = [
|
||||
records: list[dict[str, object]] = [
|
||||
{"max_tokens": 100, "temperature": 0.7},
|
||||
{"max_tokens": 200, "temperature": 0.9},
|
||||
]
|
||||
@@ -221,7 +221,7 @@ class TestParameterSweep:
|
||||
|
||||
def test_mixed_benchmark_names_allowed(self):
|
||||
"""Test that mixing records with and without _benchmark_name is allowed."""
|
||||
records = [
|
||||
records: list[dict[str, object]] = [
|
||||
{"_benchmark_name": "exp1", "max_tokens": 100},
|
||||
{"max_tokens": 200, "temperature": 0.9},
|
||||
]
|
||||
|
||||
@@ -13,6 +13,7 @@ import soundfile as sf
|
||||
import vllm.benchmarks.datasets.datasets as datasets_module
|
||||
import vllm.benchmarks.lib.endpoint_request_func as request_func_module
|
||||
from vllm.benchmarks.lib.endpoint_request_func import RequestFuncInput
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
|
||||
@@ -44,6 +45,10 @@ class _CohereNameOnlyTokenizer(_Tokenizer):
|
||||
super().__init__("cohere/some-local-checkpoint")
|
||||
|
||||
|
||||
def _as_tokenizer(tokenizer: _Tokenizer) -> TokenizerLike:
|
||||
return cast(TokenizerLike, tokenizer)
|
||||
|
||||
|
||||
def _write_wav(path: Path, duration_s: float = 0.1, sample_rate: int = 16_000) -> None:
|
||||
num_samples = int(duration_s * sample_rate)
|
||||
sf.write(path, np.zeros(num_samples, dtype=np.float32), sample_rate)
|
||||
@@ -109,7 +114,7 @@ def test_asr_dataset_sample_handles_local_audio_paths(tmp_path: Path) -> None:
|
||||
]
|
||||
|
||||
samples = dataset.sample(
|
||||
tokenizer=_Tokenizer(),
|
||||
tokenizer=_as_tokenizer(_Tokenizer()),
|
||||
num_requests=1,
|
||||
output_len=32,
|
||||
asr_min_audio_len_sec=0.0,
|
||||
@@ -146,7 +151,7 @@ def test_asr_dataset_sample_handles_embedded_audio_bytes(
|
||||
]
|
||||
|
||||
samples = dataset.sample(
|
||||
tokenizer=_Tokenizer(),
|
||||
tokenizer=_as_tokenizer(_Tokenizer()),
|
||||
num_requests=1,
|
||||
output_len=32,
|
||||
asr_min_audio_len_sec=0.0,
|
||||
@@ -240,7 +245,7 @@ def _make_asr_dataset(tmp_path: Path) -> datasets_module.ASRDataset:
|
||||
def test_asr_dataset_cohere_class_name_gets_decoder_prompt(tmp_path: Path) -> None:
|
||||
dataset = _make_asr_dataset(tmp_path)
|
||||
samples = dataset.sample(
|
||||
tokenizer=CohereAsrTokenizer(),
|
||||
tokenizer=_as_tokenizer(CohereAsrTokenizer()),
|
||||
num_requests=1,
|
||||
output_len=32,
|
||||
asr_min_audio_len_sec=0.0,
|
||||
@@ -255,7 +260,7 @@ def test_asr_dataset_cohere_name_or_path_fallback_gets_decoder_prompt(
|
||||
) -> None:
|
||||
dataset = _make_asr_dataset(tmp_path)
|
||||
samples = dataset.sample(
|
||||
tokenizer=_CohereNameOnlyTokenizer(),
|
||||
tokenizer=_as_tokenizer(_CohereNameOnlyTokenizer()),
|
||||
num_requests=1,
|
||||
output_len=32,
|
||||
asr_min_audio_len_sec=0.0,
|
||||
@@ -268,7 +273,7 @@ def test_asr_dataset_cohere_name_or_path_fallback_gets_decoder_prompt(
|
||||
def test_asr_dataset_unknown_tokenizer_gets_empty_prompt(tmp_path: Path) -> None:
|
||||
dataset = _make_asr_dataset(tmp_path)
|
||||
samples = dataset.sample(
|
||||
tokenizer=_Tokenizer(name_or_path="some-other/asr-model"),
|
||||
tokenizer=_as_tokenizer(_Tokenizer(name_or_path="some-other/asr-model")),
|
||||
num_requests=1,
|
||||
output_len=32,
|
||||
asr_min_audio_len_sec=0.0,
|
||||
|
||||
@@ -271,7 +271,7 @@ def test_bfcl_prompt_len_includes_tools(tmp_path: Path) -> None:
|
||||
args.num_prompts = 1
|
||||
|
||||
with _patch_hf_api(fake_download):
|
||||
samples = get_samples(args, fake)
|
||||
samples = get_samples(args, fake) # type: ignore[arg-type]
|
||||
|
||||
assert len(samples) == 1
|
||||
assert captured["tools"] is not None, (
|
||||
@@ -311,7 +311,7 @@ def test_bfcl_prompt_len_falls_back_when_tokenizer_rejects_tools(
|
||||
args.num_prompts = 1
|
||||
|
||||
with _patch_hf_api(fake_download):
|
||||
samples = get_samples(args, _LegacyTokenizer())
|
||||
samples = get_samples(args, _LegacyTokenizer()) # type: ignore[arg-type]
|
||||
|
||||
assert len(samples) == 1
|
||||
assert samples[0].prompt_len > 0
|
||||
|
||||
@@ -58,7 +58,7 @@ def test_chat_template_kwargs_forwarded(tmp_path: Path) -> None:
|
||||
_write_one(jsonl)
|
||||
|
||||
tok = _RecordingTokenizer()
|
||||
get_samples(_args(str(jsonl), {"thinking": True}), tok)
|
||||
get_samples(_args(str(jsonl), {"thinking": True}), tok) # type: ignore[arg-type]
|
||||
|
||||
assert tok.captured_kwargs == {"thinking": True}
|
||||
|
||||
@@ -70,6 +70,6 @@ def test_chat_template_kwargs_default_is_noop(tmp_path: Path) -> None:
|
||||
_write_one(jsonl)
|
||||
|
||||
tok = _RecordingTokenizer()
|
||||
get_samples(_args(str(jsonl), None), tok)
|
||||
get_samples(_args(str(jsonl), None), tok) # type: ignore[arg-type]
|
||||
|
||||
assert tok.captured_kwargs == {}
|
||||
|
||||
@@ -4,7 +4,7 @@ import json
|
||||
from argparse import Namespace
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from typing import Any, cast
|
||||
|
||||
import pybase64 as base64
|
||||
import pytest
|
||||
@@ -17,6 +17,7 @@ from vllm.benchmarks.lib.endpoint_request_func import (
|
||||
_get_chat_content,
|
||||
_get_chat_messages,
|
||||
)
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
|
||||
@@ -31,6 +32,10 @@ class _Tokenizer:
|
||||
return _TokenizedPrompt(prompt)
|
||||
|
||||
|
||||
def _tokenizer() -> TokenizerLike:
|
||||
return cast(TokenizerLike, _Tokenizer())
|
||||
|
||||
|
||||
def _write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None:
|
||||
with path.open("w") as f:
|
||||
for row in rows:
|
||||
@@ -92,7 +97,7 @@ def test_get_samples_custom_image_cli_path_supports_multi_image_and_content(
|
||||
],
|
||||
)
|
||||
|
||||
samples = get_samples(_args_for_custom_image(jsonl), _Tokenizer())
|
||||
samples = get_samples(_args_for_custom_image(jsonl), _tokenizer())
|
||||
|
||||
assert len(samples) == 2
|
||||
assert samples[0].request_id == "req-0"
|
||||
@@ -106,6 +111,7 @@ def test_get_samples_custom_image_cli_path_supports_multi_image_and_content(
|
||||
assert samples[1].multi_modal_data is None
|
||||
assert isinstance(samples[1].prompt, list)
|
||||
assert samples[1].prompt[0] == {"type": "text", "text": "Now compare "}
|
||||
assert isinstance(samples[1].prompt[1], dict)
|
||||
assert samples[1].prompt[1]["image_url"]["url"] == f"file://{image_c}"
|
||||
|
||||
|
||||
@@ -126,7 +132,7 @@ def test_custom_image_dataset_uses_all_image_files(tmp_path: Path) -> None:
|
||||
|
||||
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
|
||||
samples = dataset.sample(
|
||||
tokenizer=_Tokenizer(),
|
||||
tokenizer=_tokenizer(),
|
||||
num_requests=1,
|
||||
output_len=32,
|
||||
)
|
||||
@@ -171,7 +177,7 @@ def test_custom_image_dataset_preserves_interleaved_content_order(
|
||||
|
||||
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
|
||||
samples = dataset.sample(
|
||||
tokenizer=_Tokenizer(),
|
||||
tokenizer=_tokenizer(),
|
||||
num_requests=1,
|
||||
output_len=32,
|
||||
)
|
||||
@@ -181,14 +187,18 @@ def test_custom_image_dataset_preserves_interleaved_content_order(
|
||||
assert sample.multi_modal_data is None
|
||||
assert sample.prompt_len == 2
|
||||
assert isinstance(sample.prompt, list)
|
||||
assert [part["type"] for part in sample.prompt] == [
|
||||
parts = []
|
||||
for part in sample.prompt:
|
||||
assert isinstance(part, dict)
|
||||
parts.append(part)
|
||||
assert [part["type"] for part in parts] == [
|
||||
"text",
|
||||
"image_url",
|
||||
"text",
|
||||
"image_url",
|
||||
]
|
||||
assert sample.prompt[1]["image_url"]["url"] == f"file://{image_a}"
|
||||
assert sample.prompt[3]["image_url"] == {
|
||||
assert parts[1]["image_url"]["url"] == f"file://{image_a}"
|
||||
assert parts[3]["image_url"] == {
|
||||
"url": f"file://{image_b}",
|
||||
"detail": "low",
|
||||
}
|
||||
@@ -223,7 +233,7 @@ def test_custom_image_dataset_wraps_interleaved_content_for_multimodal_chat(
|
||||
|
||||
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
|
||||
samples = dataset.sample(
|
||||
tokenizer=_Tokenizer(),
|
||||
tokenizer=_tokenizer(),
|
||||
num_requests=1,
|
||||
output_len=32,
|
||||
enable_multimodal_chat=True,
|
||||
@@ -289,7 +299,7 @@ def test_custom_image_dataset_encodes_image_media_when_requested(
|
||||
|
||||
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
|
||||
samples = dataset.sample(
|
||||
tokenizer=_Tokenizer(),
|
||||
tokenizer=_tokenizer(),
|
||||
num_requests=1,
|
||||
output_len=32,
|
||||
ensure_client_side_data=True,
|
||||
@@ -335,7 +345,7 @@ def test_custom_image_dataset_encodes_interleaved_image_media(
|
||||
|
||||
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
|
||||
samples = dataset.sample(
|
||||
tokenizer=_Tokenizer(),
|
||||
tokenizer=_tokenizer(),
|
||||
num_requests=1,
|
||||
output_len=32,
|
||||
ensure_client_side_data=True,
|
||||
@@ -343,9 +353,13 @@ def test_custom_image_dataset_encodes_interleaved_image_media(
|
||||
|
||||
sample = samples[0]
|
||||
assert isinstance(sample.prompt, list)
|
||||
_assert_png_data_url(sample.prompt[1]["image_url"]["url"])
|
||||
_assert_png_data_url(sample.prompt[2]["image_url"]["url"])
|
||||
assert sample.prompt[2]["image_url"]["detail"] == "low"
|
||||
parts = []
|
||||
for part in sample.prompt:
|
||||
assert isinstance(part, dict)
|
||||
parts.append(part)
|
||||
_assert_png_data_url(parts[1]["image_url"]["url"])
|
||||
_assert_png_data_url(parts[2]["image_url"]["url"])
|
||||
assert parts[2]["image_url"]["detail"] == "low"
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
@@ -363,7 +377,7 @@ def test_custom_image_dataset_rejects_invalid_image_media(
|
||||
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
|
||||
with pytest.raises(ValueError, match="Invalid image URL"):
|
||||
dataset.sample(
|
||||
tokenizer=_Tokenizer(),
|
||||
tokenizer=_tokenizer(),
|
||||
num_requests=1,
|
||||
output_len=32,
|
||||
ensure_client_side_data=True,
|
||||
@@ -380,7 +394,7 @@ def test_custom_image_dataset_rejects_invalid_content_part(
|
||||
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
|
||||
with pytest.raises(ValueError, match="type 'text', 'image', or 'image_url'"):
|
||||
dataset.sample(
|
||||
tokenizer=_Tokenizer(),
|
||||
tokenizer=_tokenizer(),
|
||||
num_requests=1,
|
||||
output_len=32,
|
||||
)
|
||||
|
||||
@@ -37,6 +37,7 @@ def random_dataset_params() -> Params:
|
||||
|
||||
def _fingerprint_sample(req: SampleRequest) -> tuple[str, int, int]:
|
||||
"""Project a SampleRequest into a comparable tuple."""
|
||||
assert isinstance(req.prompt, str)
|
||||
return (req.prompt, req.prompt_len, req.expected_output_len)
|
||||
|
||||
|
||||
@@ -153,6 +154,7 @@ def _mm_fingerprint_sample(
|
||||
- count of multimodal items
|
||||
- per-item type and URL prefix (e.g., 'data:image/jpeg;base64,')
|
||||
"""
|
||||
assert isinstance(req.prompt, str)
|
||||
items = req.multi_modal_data or []
|
||||
item_prefixes: list[str] = []
|
||||
for it in items:
|
||||
|
||||
@@ -19,7 +19,7 @@ class _FakeTokenizer(TokenizerLike):
|
||||
# -- Properties required by TokenizerLike --
|
||||
|
||||
@classmethod
|
||||
def from_pretrained(cls, path_or_repo_id, *a, **kw): # type: ignore[override]
|
||||
def from_pretrained(cls, path_or_repo_id, *a, **kw):
|
||||
return cls()
|
||||
|
||||
@property
|
||||
@@ -80,16 +80,16 @@ class _FakeTokenizer(TokenizerLike):
|
||||
def apply_chat_template(self, messages, **kw): # type: ignore[override]
|
||||
raise NotImplementedError
|
||||
|
||||
def convert_tokens_to_ids(self, tokens): # type: ignore[override]
|
||||
def convert_tokens_to_ids(self, tokens):
|
||||
raise NotImplementedError
|
||||
|
||||
def convert_tokens_to_string(self, tokens: list[str]) -> str:
|
||||
raise NotImplementedError
|
||||
|
||||
def decode(self, ids, skip_special_tokens: bool = False) -> str: # type: ignore[override]
|
||||
def decode(self, ids, skip_special_tokens: bool = False) -> str:
|
||||
raise NotImplementedError
|
||||
|
||||
def convert_ids_to_tokens( # type: ignore[override]
|
||||
def convert_ids_to_tokens(
|
||||
self, ids, skip_special_tokens: bool = False
|
||||
) -> list[str]:
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -9,6 +9,7 @@ serve-side dataset coverage.
|
||||
"""
|
||||
|
||||
import subprocess
|
||||
from argparse import Namespace
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
@@ -289,7 +290,7 @@ def test_to_serve_args_supports_mm_processor_flags() -> None:
|
||||
explicit 0 instead of coercing it to None (which later crashed
|
||||
RandomMultiModalDataset.get_prefix on ``None <= 0``).
|
||||
"""
|
||||
args = SimpleNamespace(
|
||||
args = Namespace(
|
||||
random_input_len=32,
|
||||
random_output_len=1,
|
||||
random_prefix_len=0,
|
||||
@@ -306,7 +307,7 @@ def test_to_serve_args_supports_mm_processor_flags() -> None:
|
||||
|
||||
def test_to_serve_args_keeps_legacy_flag_fallback() -> None:
|
||||
"""Callers that only set the legacy flags still get them mapped."""
|
||||
args = SimpleNamespace(
|
||||
args = Namespace(
|
||||
input_len=256,
|
||||
output_len=8,
|
||||
prefix_len=16,
|
||||
|
||||
@@ -355,9 +355,11 @@ class TestToolResultContent:
|
||||
if m["role"] == "user" and isinstance(m.get("content"), list)
|
||||
]
|
||||
assert len(follow_up) == 1
|
||||
assert follow_up[0]["content"][0]["image_url"]["url"] == (
|
||||
"data:image/jpeg;base64,QUFB"
|
||||
)
|
||||
content = follow_up[0]["content"]
|
||||
assert isinstance(content, list)
|
||||
image = content[0]
|
||||
assert image["type"] == "image_url"
|
||||
assert image["image_url"]["url"] == "data:image/jpeg;base64,QUFB"
|
||||
|
||||
def test_tool_result_with_multiple_images(self):
|
||||
request = self._make_tool_result_request(
|
||||
@@ -387,7 +389,12 @@ class TestToolResultContent:
|
||||
if m["role"] == "user" and isinstance(m.get("content"), list)
|
||||
]
|
||||
assert len(follow_up) == 1
|
||||
urls = [p["image_url"]["url"] for p in follow_up[0]["content"]]
|
||||
content = follow_up[0]["content"]
|
||||
assert isinstance(content, list)
|
||||
urls = []
|
||||
for part in content:
|
||||
assert part["type"] == "image_url"
|
||||
urls.append(part["image_url"]["url"])
|
||||
assert urls == [
|
||||
"data:image/png;base64,IMG1",
|
||||
"https://example.com/img2.jpg",
|
||||
@@ -584,7 +591,8 @@ class TestThinkingBlockConversion:
|
||||
asst = asst_msgs[0]
|
||||
|
||||
assert asst.get("reasoning") == "I need to call the calculator."
|
||||
tool_calls = list(asst.get("tool_calls", []))
|
||||
tool_calls = asst.get("tool_calls")
|
||||
assert tool_calls is not None
|
||||
assert len(tool_calls) == 1
|
||||
assert tool_calls[0]["function"]["name"] == "calculator"
|
||||
# No text content alongside reasoning + tool_use.
|
||||
@@ -1439,7 +1447,7 @@ class TestDetectMergeInlineSystem:
|
||||
model_config=None,
|
||||
)
|
||||
assert (
|
||||
AnthropicServingMessages._should_merge_inline_system(online_renderer)
|
||||
AnthropicServingMessages._should_merge_inline_system(online_renderer) # type: ignore[arg-type]
|
||||
is expected
|
||||
)
|
||||
assert mock_logger.warning_once.called is expected
|
||||
|
||||
@@ -7,22 +7,26 @@ from unittest.mock import Mock
|
||||
import pytest
|
||||
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
|
||||
from vllm.inputs import TextPrompt
|
||||
from vllm.outputs import RequestOutput
|
||||
|
||||
|
||||
def _make_mock_llm() -> LLM:
|
||||
llm = object.__new__(LLM)
|
||||
llm.model_config = SimpleNamespace(
|
||||
llm.model_config = SimpleNamespace( # type: ignore[assignment]
|
||||
runner_type="generate", enable_prompt_embeds=False
|
||||
)
|
||||
return llm
|
||||
|
||||
|
||||
def test_generate_forwards_mm_processor_kwargs() -> None:
|
||||
def test_generate_forwards_mm_processor_kwargs(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
llm = _make_mock_llm()
|
||||
mm_processor_kwargs = {"num_crops": 4}
|
||||
sampling_params = SamplingParams(max_tokens=1)
|
||||
|
||||
llm._run_completion = Mock(return_value=["ok"])
|
||||
_run_completion = Mock(return_value=["ok"])
|
||||
monkeypatch.setattr(llm, "_run_completion", _run_completion)
|
||||
|
||||
outputs = llm.generate(
|
||||
"prompt",
|
||||
@@ -31,17 +35,18 @@ def test_generate_forwards_mm_processor_kwargs() -> None:
|
||||
)
|
||||
|
||||
assert outputs == ["ok"]
|
||||
assert llm._run_completion.call_args.kwargs["mm_processor_kwargs"] == (
|
||||
assert _run_completion.call_args.kwargs["mm_processor_kwargs"] == (
|
||||
mm_processor_kwargs
|
||||
)
|
||||
|
||||
|
||||
def test_enqueue_forwards_mm_processor_kwargs() -> None:
|
||||
def test_enqueue_forwards_mm_processor_kwargs(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
llm = _make_mock_llm()
|
||||
mm_processor_kwargs = {"do_resize": False}
|
||||
sampling_params = SamplingParams(max_tokens=1)
|
||||
|
||||
llm._add_completion_requests = Mock(return_value=["req-0"])
|
||||
_add_completion_requests = Mock(return_value=["req-0"])
|
||||
monkeypatch.setattr(llm, "_add_completion_requests", _add_completion_requests)
|
||||
|
||||
request_ids = llm.enqueue(
|
||||
"prompt",
|
||||
@@ -51,18 +56,19 @@ def test_enqueue_forwards_mm_processor_kwargs() -> None:
|
||||
)
|
||||
|
||||
assert request_ids == ["req-0"]
|
||||
assert llm._add_completion_requests.call_args.kwargs["mm_processor_kwargs"] == (
|
||||
assert _add_completion_requests.call_args.kwargs["mm_processor_kwargs"] == (
|
||||
mm_processor_kwargs
|
||||
)
|
||||
|
||||
|
||||
def test_chat_forwards_mm_processor_kwargs() -> None:
|
||||
def test_chat_forwards_mm_processor_kwargs(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
llm = _make_mock_llm()
|
||||
mm_processor_kwargs = {"do_pan_and_scan": True}
|
||||
sampling_params = SamplingParams(max_tokens=1)
|
||||
messages = [{"role": "user", "content": "hello"}]
|
||||
messages: list[ChatCompletionMessageParam] = [{"role": "user", "content": "hello"}]
|
||||
|
||||
llm._run_chat = Mock(return_value=["ok"])
|
||||
_run_chat = Mock(return_value=["ok"])
|
||||
monkeypatch.setattr(llm, "_run_chat", _run_chat)
|
||||
|
||||
outputs = llm.chat(
|
||||
messages,
|
||||
@@ -71,18 +77,19 @@ def test_chat_forwards_mm_processor_kwargs() -> None:
|
||||
)
|
||||
|
||||
assert outputs == ["ok"]
|
||||
assert llm._run_chat.call_args.kwargs["mm_processor_kwargs"] == (
|
||||
mm_processor_kwargs
|
||||
)
|
||||
assert _run_chat.call_args.kwargs["mm_processor_kwargs"] == mm_processor_kwargs
|
||||
|
||||
|
||||
def test_enqueue_chat_forwards_mm_processor_kwargs() -> None:
|
||||
def test_enqueue_chat_forwards_mm_processor_kwargs(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
llm = _make_mock_llm()
|
||||
mm_processor_kwargs = {"do_pan_and_scan": True}
|
||||
sampling_params = SamplingParams(max_tokens=1)
|
||||
messages = [{"role": "user", "content": "hello"}]
|
||||
messages: list[ChatCompletionMessageParam] = [{"role": "user", "content": "hello"}]
|
||||
|
||||
llm._add_chat_requests = Mock(return_value=["req-0"])
|
||||
_add_chat_requests = Mock(return_value=["req-0"])
|
||||
monkeypatch.setattr(llm, "_add_chat_requests", _add_chat_requests)
|
||||
|
||||
request_ids = llm.enqueue_chat(
|
||||
messages,
|
||||
@@ -92,67 +99,79 @@ def test_enqueue_chat_forwards_mm_processor_kwargs() -> None:
|
||||
)
|
||||
|
||||
assert request_ids == ["req-0"]
|
||||
assert llm._add_chat_requests.call_args.kwargs["mm_processor_kwargs"] == (
|
||||
assert _add_chat_requests.call_args.kwargs["mm_processor_kwargs"] == (
|
||||
mm_processor_kwargs
|
||||
)
|
||||
|
||||
|
||||
def test_run_chat_forwards_mm_processor_kwargs() -> None:
|
||||
def test_run_chat_forwards_mm_processor_kwargs(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
llm = _make_mock_llm()
|
||||
mm_processor_kwargs = {"num_crops": 8}
|
||||
sampling_params = SamplingParams(max_tokens=1)
|
||||
messages = [{"role": "user", "content": "hello"}]
|
||||
messages: list[ChatCompletionMessageParam] = [{"role": "user", "content": "hello"}]
|
||||
sentinel_output = ["done"]
|
||||
|
||||
llm._add_chat_requests = Mock()
|
||||
llm._run_engine = Mock(return_value=sentinel_output)
|
||||
_add_chat_requests = Mock()
|
||||
monkeypatch.setattr(llm, "_add_chat_requests", _add_chat_requests)
|
||||
_run_engine = Mock(return_value=sentinel_output)
|
||||
monkeypatch.setattr(llm, "_run_engine", _run_engine)
|
||||
|
||||
outputs = llm._run_chat(
|
||||
messages=messages,
|
||||
params=sampling_params,
|
||||
output_type=object,
|
||||
output_type=RequestOutput,
|
||||
use_tqdm=False,
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
)
|
||||
|
||||
assert outputs == sentinel_output
|
||||
assert llm._add_chat_requests.call_args.kwargs["mm_processor_kwargs"] == (
|
||||
assert _add_chat_requests.call_args.kwargs["mm_processor_kwargs"] == (
|
||||
mm_processor_kwargs
|
||||
)
|
||||
|
||||
|
||||
def test_run_completion_forwards_mm_processor_kwargs() -> None:
|
||||
def test_run_completion_forwards_mm_processor_kwargs(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
llm = _make_mock_llm()
|
||||
mm_processor_kwargs = {"min_pixels": 4 * 28 * 28}
|
||||
sampling_params = SamplingParams(max_tokens=1)
|
||||
sentinel_output = ["done"]
|
||||
|
||||
llm._add_completion_requests = Mock()
|
||||
llm._run_engine = Mock(return_value=sentinel_output)
|
||||
_add_completion_requests = Mock()
|
||||
monkeypatch.setattr(llm, "_add_completion_requests", _add_completion_requests)
|
||||
_run_engine = Mock(return_value=sentinel_output)
|
||||
monkeypatch.setattr(llm, "_run_engine", _run_engine)
|
||||
|
||||
outputs = llm._run_completion(
|
||||
prompts=["prompt"],
|
||||
params=sampling_params,
|
||||
output_type=object,
|
||||
output_type=RequestOutput,
|
||||
use_tqdm=False,
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
)
|
||||
|
||||
assert outputs == sentinel_output
|
||||
assert llm._add_completion_requests.call_args.kwargs["mm_processor_kwargs"] == (
|
||||
assert _add_completion_requests.call_args.kwargs["mm_processor_kwargs"] == (
|
||||
mm_processor_kwargs
|
||||
)
|
||||
|
||||
|
||||
def test_add_completion_requests_forwards_mm_processor_kwargs() -> None:
|
||||
def test_add_completion_requests_forwards_mm_processor_kwargs(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
llm = _make_mock_llm()
|
||||
mm_processor_kwargs = {"max_dynamic_patch": 4}
|
||||
sampling_params = SamplingParams(max_tokens=1)
|
||||
|
||||
llm._params_to_seq = Mock(return_value=[sampling_params])
|
||||
llm._lora_request_to_seq = Mock(return_value=[None])
|
||||
llm._priority_to_seq = Mock(return_value=[0])
|
||||
llm._preprocess_cmpl_one = Mock(return_value={"prompt_token_ids": [1]})
|
||||
_params_to_seq = Mock(return_value=[sampling_params])
|
||||
monkeypatch.setattr(llm, "_params_to_seq", _params_to_seq)
|
||||
_lora_request_to_seq = Mock(return_value=[None])
|
||||
monkeypatch.setattr(llm, "_lora_request_to_seq", _lora_request_to_seq)
|
||||
_priority_to_seq = Mock(return_value=[0])
|
||||
monkeypatch.setattr(llm, "_priority_to_seq", _priority_to_seq)
|
||||
_preprocess_cmpl_one = Mock(return_value={"prompt_token_ids": [1]})
|
||||
monkeypatch.setattr(llm, "_preprocess_cmpl_one", _preprocess_cmpl_one)
|
||||
|
||||
captured_prompts = []
|
||||
|
||||
@@ -160,7 +179,8 @@ def test_add_completion_requests_forwards_mm_processor_kwargs() -> None:
|
||||
captured_prompts.extend(prompts)
|
||||
return ["req-0"]
|
||||
|
||||
llm._render_and_add_requests = Mock(side_effect=fake_render_and_add_requests)
|
||||
_render_and_add_requests = Mock(side_effect=fake_render_and_add_requests)
|
||||
monkeypatch.setattr(llm, "_render_and_add_requests", _render_and_add_requests)
|
||||
|
||||
request_ids = llm._add_completion_requests(
|
||||
prompts=["prompt"],
|
||||
@@ -170,7 +190,7 @@ def test_add_completion_requests_forwards_mm_processor_kwargs() -> None:
|
||||
)
|
||||
|
||||
assert request_ids == ["req-0"]
|
||||
llm._preprocess_cmpl_one.assert_called_once_with(
|
||||
_preprocess_cmpl_one.assert_called_once_with(
|
||||
"prompt",
|
||||
None,
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
@@ -183,7 +203,7 @@ def test_preprocess_cmpl_applies_mm_processor_kwargs_to_renderer(
|
||||
) -> None:
|
||||
llm = _make_mock_llm()
|
||||
mm_processor_kwargs = {"num_crops": 8}
|
||||
prompt = {"prompt": "<image>", "multi_modal_data": {"image": object()}}
|
||||
prompt: TextPrompt = {"prompt": "<image>", "multi_modal_data": {"image": object()}}
|
||||
|
||||
renderer = Mock()
|
||||
renderer.default_cmpl_tok_params = Mock()
|
||||
@@ -213,7 +233,7 @@ def test_preprocess_cmpl_keeps_prompt_mm_processor_kwargs_when_no_override(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
llm = _make_mock_llm()
|
||||
prompt = {
|
||||
prompt: TextPrompt = {
|
||||
"prompt": "<image>",
|
||||
"multi_modal_data": {"image": object()},
|
||||
"mm_processor_kwargs": {"num_crops": 2},
|
||||
@@ -243,7 +263,9 @@ def test_preprocess_cmpl_keeps_prompt_mm_processor_kwargs_when_no_override(
|
||||
def test_preprocess_chat_applies_mm_processor_kwargs_to_renderer() -> None:
|
||||
llm = _make_mock_llm()
|
||||
mm_processor_kwargs = {"num_crops": 8}
|
||||
messages = [[{"role": "user", "content": "Describe this image."}]]
|
||||
messages: list[list[ChatCompletionMessageParam]] = [
|
||||
[{"role": "user", "content": "Describe this image."}]
|
||||
]
|
||||
|
||||
renderer = Mock()
|
||||
renderer.tokenizer = object()
|
||||
@@ -269,7 +291,9 @@ def test_preprocess_chat_applies_mm_processor_kwargs_to_renderer() -> None:
|
||||
|
||||
def test_preprocess_chat_omits_mm_processor_kwargs_when_no_override() -> None:
|
||||
llm = _make_mock_llm()
|
||||
messages = [[{"role": "user", "content": "Describe this image."}]]
|
||||
messages: list[list[ChatCompletionMessageParam]] = [
|
||||
[{"role": "user", "content": "Describe this image."}]
|
||||
]
|
||||
|
||||
renderer = Mock()
|
||||
renderer.tokenizer = object()
|
||||
@@ -291,7 +315,9 @@ def test_preprocess_chat_omits_mm_processor_kwargs_when_no_override() -> None:
|
||||
def test_preprocess_chat_defaults_add_special_tokens_to_false() -> None:
|
||||
# Matches `ChatCompletionRequest.add_special_tokens` on the server.
|
||||
llm = _make_mock_llm()
|
||||
messages = [[{"role": "user", "content": "hi"}]]
|
||||
messages: list[list[ChatCompletionMessageParam]] = [
|
||||
[{"role": "user", "content": "hi"}]
|
||||
]
|
||||
|
||||
renderer = Mock()
|
||||
renderer.tokenizer = object()
|
||||
@@ -309,7 +335,9 @@ def test_preprocess_chat_defaults_add_special_tokens_to_false() -> None:
|
||||
|
||||
def test_preprocess_chat_tokenization_kwargs_override_add_special_tokens() -> None:
|
||||
llm = _make_mock_llm()
|
||||
messages = [[{"role": "user", "content": "hi"}]]
|
||||
messages: list[list[ChatCompletionMessageParam]] = [
|
||||
[{"role": "user", "content": "hi"}]
|
||||
]
|
||||
|
||||
renderer = Mock()
|
||||
renderer.tokenizer = object()
|
||||
|
||||
@@ -47,18 +47,20 @@ async def client(server):
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def base64_encoded_audio() -> dict[str, str]:
|
||||
return {
|
||||
audio_url: encode_audio_base64(*fetch_audio(audio_url))
|
||||
for audio_url in TEST_AUDIO_URLS
|
||||
}
|
||||
encoded = {}
|
||||
for audio_url in TEST_AUDIO_URLS:
|
||||
audio, sample_rate = fetch_audio(audio_url)
|
||||
encoded[audio_url] = encode_audio_base64(audio, int(sample_rate))
|
||||
return encoded
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def url_encoded_audio() -> dict[str, str]:
|
||||
return {
|
||||
audio_url: encode_audio_url(*fetch_audio(audio_url))
|
||||
for audio_url in TEST_AUDIO_URLS
|
||||
}
|
||||
encoded = {}
|
||||
for audio_url in TEST_AUDIO_URLS:
|
||||
audio, sample_rate = fetch_audio(audio_url)
|
||||
encoded[audio_url] = encode_audio_url(audio, int(sample_rate))
|
||||
return encoded
|
||||
|
||||
|
||||
def dummy_messages_from_audio_url(
|
||||
|
||||
@@ -7,6 +7,7 @@ import torch
|
||||
|
||||
from tests.models.utils import softmax
|
||||
from vllm import LLM, ClassificationRequestOutput, PoolingParams
|
||||
from vllm.inputs import PromptType
|
||||
from vllm.tasks import PoolingTask
|
||||
|
||||
MODEL_NAME = "jason9693/Qwen2.5-1.5B-apeach"
|
||||
@@ -53,7 +54,8 @@ def test_token_ids_prompts(llm: LLM):
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_list_prompts(llm: LLM):
|
||||
outputs = llm.classify([prompt, prompt_token_ids], use_tqdm=False)
|
||||
prompts: list[PromptType] = [prompt, prompt_token_ids]
|
||||
outputs = llm.classify(prompts, use_tqdm=False)
|
||||
assert len(outputs) == 2
|
||||
for i in range(len(outputs)):
|
||||
assert isinstance(outputs[i], ClassificationRequestOutput)
|
||||
|
||||
@@ -21,6 +21,7 @@ from vllm.entrypoints.pooling.typing import (
|
||||
PoolingEngineInput,
|
||||
PoolingServeContext,
|
||||
)
|
||||
from vllm.inputs import tokens_input
|
||||
from vllm.outputs import PoolingOutput, PoolingRequestOutput
|
||||
|
||||
|
||||
@@ -28,7 +29,7 @@ class TestEmbeddingRequestParsing:
|
||||
"""Unit tests for OpenAI embedding request parsing."""
|
||||
|
||||
def test_input_messages_parses_as_chat_request(self):
|
||||
request = TypeAdapter(EmbeddingRequest).validate_python(
|
||||
request: EmbeddingRequest = TypeAdapter(EmbeddingRequest).validate_python(
|
||||
{
|
||||
"model": "test",
|
||||
"input": [{"role": "user", "content": "hello"}],
|
||||
@@ -42,7 +43,7 @@ class TestEmbeddingRequestParsing:
|
||||
assert request.chat_template_kwargs == {"instruction": "Represent the query: "}
|
||||
|
||||
def test_batched_input_messages_parses_as_batch_chat_input_request(self):
|
||||
request = TypeAdapter(EmbeddingRequest).validate_python(
|
||||
request: EmbeddingRequest = TypeAdapter(EmbeddingRequest).validate_python(
|
||||
{
|
||||
"model": "test",
|
||||
"input": [
|
||||
@@ -65,7 +66,7 @@ class TestEmbeddingRequestParsing:
|
||||
assert request.chat_template_kwargs == {"instruction": "Represent the query: "}
|
||||
|
||||
def test_token_ids_still_parse_as_completion_request(self):
|
||||
request = TypeAdapter(EmbeddingRequest).validate_python(
|
||||
request: EmbeddingRequest = TypeAdapter(EmbeddingRequest).validate_python(
|
||||
{
|
||||
"model": "test",
|
||||
"input": [[1, 2, 3], [4, 5]],
|
||||
@@ -76,7 +77,7 @@ class TestEmbeddingRequestParsing:
|
||||
assert request.input == [[1, 2, 3], [4, 5]]
|
||||
|
||||
def test_messages_still_parses_as_chat_request(self):
|
||||
request = TypeAdapter(EmbeddingRequest).validate_python(
|
||||
request: EmbeddingRequest = TypeAdapter(EmbeddingRequest).validate_python(
|
||||
{
|
||||
"model": "test",
|
||||
"messages": [{"role": "user", "content": "hello"}],
|
||||
@@ -89,7 +90,7 @@ class TestEmbeddingRequestParsing:
|
||||
assert request.chat_template_kwargs == {"instruction": "Represent the query: "}
|
||||
|
||||
def test_batched_messages_parses_as_batch_chat_request(self):
|
||||
request = TypeAdapter(EmbeddingRequest).validate_python(
|
||||
request: EmbeddingRequest = TypeAdapter(EmbeddingRequest).validate_python(
|
||||
{
|
||||
"model": "test",
|
||||
"messages": [
|
||||
@@ -410,13 +411,13 @@ class TestChunkedEmbeddingProcessing:
|
||||
@classmethod
|
||||
def _make_handler(cls):
|
||||
handler = object.__new__(EmbedIOProcessor)
|
||||
handler.model_config = cls._FakeModelConfig()
|
||||
handler.model_config = cls._FakeModelConfig() # type: ignore[assignment]
|
||||
handler.enable_chunked_processing = True
|
||||
return handler
|
||||
|
||||
@staticmethod
|
||||
def _make_context() -> PoolingServeContext[EmbeddingCompletionRequest]:
|
||||
request = TypeAdapter(EmbeddingRequest).validate_python(
|
||||
request: EmbeddingRequest = TypeAdapter(EmbeddingRequest).validate_python(
|
||||
{
|
||||
"model": "test",
|
||||
"input": [[0, 1, 2, 3, 4], [10, 11]],
|
||||
@@ -431,13 +432,13 @@ class TestChunkedEmbeddingProcessing:
|
||||
request_id="embd-client-prompt-999-chunk-888",
|
||||
engine_inputs=[
|
||||
PoolingEngineInput(
|
||||
prompts={"prompt_token_ids": [0, 1, 2, 3, 4]},
|
||||
prompts=tokens_input(prompt_token_ids=[0, 1, 2, 3, 4]),
|
||||
params=pooling_params,
|
||||
lora_requests=None,
|
||||
priorities=0,
|
||||
),
|
||||
PoolingEngineInput(
|
||||
prompts={"prompt_token_ids": [10, 11]},
|
||||
prompts=tokens_input(prompt_token_ids=[10, 11]),
|
||||
params=pooling_params,
|
||||
lora_requests=None,
|
||||
priorities=0,
|
||||
|
||||
@@ -7,6 +7,7 @@ import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm import LLM, EmbeddingRequestOutput, PoolingParams
|
||||
from vllm.inputs import PromptType
|
||||
from vllm.tasks import PoolingTask
|
||||
|
||||
MODEL_NAME = "intfloat/multilingual-e5-small"
|
||||
@@ -54,7 +55,8 @@ def test_token_ids_prompts(llm: LLM):
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_list_prompts(llm: LLM):
|
||||
outputs = llm.embed([prompt, prompt_token_ids], use_tqdm=False)
|
||||
prompts: list[PromptType] = [prompt, prompt_token_ids]
|
||||
outputs = llm.embed(prompts, use_tqdm=False)
|
||||
assert len(outputs) == 2
|
||||
for i in range(len(outputs)):
|
||||
assert isinstance(outputs[i], EmbeddingRequestOutput)
|
||||
|
||||
@@ -12,9 +12,13 @@ import requests
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from tests.models.language.pooling.embed_utils import run_embedding_correctness_test
|
||||
from tests.models.language.pooling.embed_utils import (
|
||||
float_embeddings,
|
||||
run_embedding_correctness_test,
|
||||
)
|
||||
from tests.models.utils import check_embeddings_close
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
|
||||
from vllm.entrypoints.pooling.embed.protocol import EmbeddingResponse
|
||||
from vllm.entrypoints.pooling.pooling.protocol import PoolingResponse
|
||||
from vllm.entrypoints.pooling.utils import (
|
||||
@@ -145,7 +149,7 @@ async def test_completion_request(
|
||||
assert embeddings.usage.prompt_tokens == len(input_tokens)
|
||||
assert embeddings.usage.total_tokens == len(input_tokens)
|
||||
|
||||
vllm_outputs = [d.embedding for d in embeddings.data]
|
||||
vllm_outputs = float_embeddings(embeddings)
|
||||
run_embedding_correctness_test(hf_model, [input_text], vllm_outputs)
|
||||
|
||||
# test input: list[int]
|
||||
@@ -165,7 +169,7 @@ async def test_completion_request(
|
||||
assert embeddings.usage.prompt_tokens == len(input_tokens)
|
||||
assert embeddings.usage.total_tokens == len(input_tokens)
|
||||
|
||||
vllm_outputs = [d.embedding for d in embeddings.data]
|
||||
vllm_outputs = float_embeddings(embeddings)
|
||||
run_embedding_correctness_test(hf_model, [input_text], vllm_outputs)
|
||||
|
||||
|
||||
@@ -194,7 +198,7 @@ async def test_completion_request_batched(
|
||||
assert embeddings.usage.prompt_tokens == len(input_tokens) * N
|
||||
assert embeddings.usage.total_tokens == len(input_tokens) * N
|
||||
|
||||
vllm_outputs = [d.embedding for d in embeddings.data]
|
||||
vllm_outputs = float_embeddings(embeddings)
|
||||
run_embedding_correctness_test(hf_model, input_texts, vllm_outputs)
|
||||
|
||||
# test list[list[int]]
|
||||
@@ -214,7 +218,7 @@ async def test_completion_request_batched(
|
||||
assert embeddings.usage.prompt_tokens == len(input_tokens) * N
|
||||
assert embeddings.usage.total_tokens == len(input_tokens) * N
|
||||
|
||||
vllm_outputs = [d.embedding for d in embeddings.data]
|
||||
vllm_outputs = float_embeddings(embeddings)
|
||||
run_embedding_correctness_test(hf_model, input_texts, vllm_outputs)
|
||||
|
||||
|
||||
@@ -331,7 +335,7 @@ async def test_padding(client: openai.AsyncOpenAI, model_name: str):
|
||||
async def test_chat_request(
|
||||
server: RemoteOpenAIServer, client: openai.AsyncOpenAI, model_name: str
|
||||
):
|
||||
messages = [
|
||||
messages: list[ChatCompletionMessageParam] = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "The cat sat on the mat.",
|
||||
@@ -382,8 +386,8 @@ async def test_chat_request(
|
||||
assert chat_embeddings.created <= completion_embeddings.created
|
||||
# Use tolerance-based comparison for embeddings
|
||||
check_embeddings_close(
|
||||
embeddings_0_lst=[d.embedding for d in chat_embeddings.data],
|
||||
embeddings_1_lst=[d.embedding for d in completion_embeddings.data],
|
||||
embeddings_0_lst=float_embeddings(chat_embeddings),
|
||||
embeddings_1_lst=float_embeddings(completion_embeddings),
|
||||
name_0="chat",
|
||||
name_1="completion",
|
||||
)
|
||||
@@ -488,7 +492,7 @@ async def test_invocations_completion_request(
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invocations_chat_request(server: RemoteOpenAIServer):
|
||||
messages = [
|
||||
messages: list[ChatCompletionMessageParam] = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "The cat sat on the mat.",
|
||||
|
||||
@@ -6,7 +6,10 @@ import openai
|
||||
import pytest
|
||||
|
||||
from tests.conftest import HfRunner
|
||||
from tests.models.language.pooling.embed_utils import run_embedding_correctness_test
|
||||
from tests.models.language.pooling.embed_utils import (
|
||||
float_embeddings,
|
||||
run_embedding_correctness_test,
|
||||
)
|
||||
from tests.models.utils import EmbedModelInfo
|
||||
from tests.utils import ROCM_EXTRA_ARGS, RemoteOpenAIServer
|
||||
from vllm.entrypoints.pooling.embed.protocol import EmbeddingResponse
|
||||
@@ -95,7 +98,7 @@ async def test_matryoshka(
|
||||
if dimensions is not None:
|
||||
assert len(embeddings.data[0].embedding) == dimensions
|
||||
|
||||
vllm_outputs = [d.embedding for d in embeddings.data]
|
||||
vllm_outputs = float_embeddings(embeddings)
|
||||
run_embedding_correctness_test(hf_model, prompts, vllm_outputs, dimensions)
|
||||
|
||||
if model_info.is_matryoshka:
|
||||
|
||||
@@ -274,6 +274,7 @@ async def test_long_text_embedding_1500_chars(
|
||||
|
||||
# Verify embedding vector validity
|
||||
embedding_vector = embeddings.data[0].embedding
|
||||
assert isinstance(embedding_vector, list)
|
||||
assert all(isinstance(x, float) for x in embedding_vector), (
|
||||
"Embedding vector should contain floats"
|
||||
)
|
||||
@@ -322,6 +323,7 @@ async def test_long_text_embedding_2500_chars(
|
||||
|
||||
# Verify embedding vector validity
|
||||
embedding_vector = embeddings.data[0].embedding
|
||||
assert isinstance(embedding_vector, list)
|
||||
assert all(isinstance(x, float) for x in embedding_vector), (
|
||||
"Embedding vector should contain floats"
|
||||
)
|
||||
@@ -409,6 +411,7 @@ async def test_chunked_vs_normal_consistency(
|
||||
|
||||
# 验证embedding向量的有效性
|
||||
embedding_vector = embeddings.data[0].embedding
|
||||
assert isinstance(embedding_vector, list)
|
||||
assert all(isinstance(x, float) for x in embedding_vector)
|
||||
assert not all(x == 0 for x in embedding_vector)
|
||||
|
||||
@@ -438,6 +441,7 @@ async def test_chunked_processing_response_format(
|
||||
|
||||
# Verify embedding vector properties
|
||||
embedding_vector = embeddings.data[0].embedding
|
||||
assert isinstance(embedding_vector, list)
|
||||
import math
|
||||
|
||||
vector_norm = math.sqrt(sum(x * x for x in embedding_vector))
|
||||
|
||||
@@ -11,6 +11,7 @@ import torch
|
||||
|
||||
from tests.models.utils import check_embeddings_close
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
|
||||
from vllm.entrypoints.pooling.pooling.protocol import PoolingResponse
|
||||
from vllm.entrypoints.pooling.utils import (
|
||||
MetadataItem,
|
||||
@@ -137,7 +138,7 @@ def test_completion_request_batched(server: RemoteOpenAIServer, model_name: str)
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
async def test_chat_request(server: RemoteOpenAIServer, model_name: str):
|
||||
messages = [
|
||||
messages: list[ChatCompletionMessageParam] = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "The cat sat on the mat.",
|
||||
@@ -524,7 +525,7 @@ async def test_invocations_chat_request(server: RemoteOpenAIServer):
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invocations_conversation_chat_request(server: RemoteOpenAIServer):
|
||||
messages = [
|
||||
messages: list[ChatCompletionMessageParam] = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "The cat sat on the mat.",
|
||||
|
||||
@@ -47,7 +47,7 @@ def llm_reranker_processor() -> CrossEncoderIOProcessor:
|
||||
)
|
||||
|
||||
processor = CrossEncoderIOProcessor.__new__(CrossEncoderIOProcessor)
|
||||
processor.model_config = SimpleNamespace(enable_prompt_embeds=False)
|
||||
processor.model_config = SimpleNamespace(enable_prompt_embeds=False) # type: ignore[assignment]
|
||||
processor.tokenizer = TokenizersBackend(
|
||||
tokenizer_object=backend,
|
||||
unk_token="[UNK]",
|
||||
@@ -95,12 +95,16 @@ def test_token_type_ids_stay_aligned_with_a_truncated_padded_prompt():
|
||||
truncation_side="left",
|
||||
)
|
||||
|
||||
prompt = tok_params.apply_post_tokenization(tokenizer, prompt)
|
||||
processed_prompt = tok_params.apply_post_tokenization(
|
||||
tokenizer, # type: ignore[arg-type]
|
||||
prompt,
|
||||
)
|
||||
assert "prompt_token_ids" in processed_prompt
|
||||
token_type_ids = _apply_post_tokenization_to_token_type_ids(
|
||||
tokenizer, tok_params, token_type_ids
|
||||
)
|
||||
|
||||
prompt_token_ids = prompt["prompt_token_ids"]
|
||||
prompt_token_ids = processed_prompt["prompt_token_ids"]
|
||||
assert len(token_type_ids) == len(prompt_token_ids)
|
||||
|
||||
# Keeping the last 40 tokens drops the first 10 query tokens, so 10 query
|
||||
|
||||
@@ -30,12 +30,18 @@ def test_online_forwards_truncate_prompt_tokens_to_proxy(monkeypatch):
|
||||
silently disables truncate_prompt_tokens for Jina rerank/score.
|
||||
"""
|
||||
proc = JinaRankingIOProcessor.__new__(JinaRankingIOProcessor)
|
||||
proc.valid_inputs_online = MagicMock(
|
||||
return_value=ScoringData(data_1=["query"], data_2=["doc"])
|
||||
monkeypatch.setattr(
|
||||
proc,
|
||||
"valid_inputs_online",
|
||||
MagicMock(return_value=ScoringData(data_1=["query"], data_2=["doc"])),
|
||||
)
|
||||
monkeypatch.setattr(proc, "_get_token_limits", MagicMock(return_value=(0, 0)))
|
||||
monkeypatch.setattr(
|
||||
proc, "ensure_str", MagicMock(side_effect=lambda data: list(data))
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
proc, "format_docs_prompts_func", MagicMock(return_value="formatted prompt")
|
||||
)
|
||||
proc._get_token_limits = MagicMock(return_value=(0, 0))
|
||||
proc.ensure_str = MagicMock(side_effect=lambda data: list(data))
|
||||
proc.format_docs_prompts_func = MagicMock(return_value="formatted prompt")
|
||||
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
@@ -71,8 +77,10 @@ def offline_processor_and_context():
|
||||
backend.pre_tokenizer = pre_tokenizers.WhitespaceSplit()
|
||||
proc = JinaRankingIOProcessor.__new__(JinaRankingIOProcessor)
|
||||
proc.tokenizer = TokenizersBackend(tokenizer_object=backend)
|
||||
proc.model_config = SimpleNamespace(max_model_len=1024, is_encoder_decoder=False)
|
||||
proc.renderer = SimpleNamespace(
|
||||
proc.model_config = SimpleNamespace( # type: ignore[assignment]
|
||||
max_model_len=1024, is_encoder_decoder=False
|
||||
)
|
||||
proc.renderer = SimpleNamespace( # type: ignore[assignment]
|
||||
default_cmpl_tok_params=TokenizeParams(max_total_tokens=1024)
|
||||
)
|
||||
ctx = OfflineScoringInputsContext(
|
||||
|
||||
@@ -6,7 +6,7 @@ from unittest.mock import AsyncMock, Mock
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.pooling.scoring.serving import ServingScores
|
||||
from vllm.entrypoints.pooling.typing import PoolingServeContext
|
||||
from vllm.entrypoints.pooling.typing import PoolingEngineInput, PoolingServeContext
|
||||
from vllm.inputs import tokens_input
|
||||
from vllm.pooling_params import PoolingParams
|
||||
|
||||
@@ -21,8 +21,18 @@ def _make_context() -> PoolingServeContext:
|
||||
priorities=None,
|
||||
prompt_extras=None,
|
||||
engine_inputs=[
|
||||
tokens_input(prompt_token_ids=[1]),
|
||||
tokens_input(prompt_token_ids=[2]),
|
||||
PoolingEngineInput(
|
||||
prompts=tokens_input(prompt_token_ids=[1]),
|
||||
params=PoolingParams(task="token_embed"),
|
||||
lora_requests=None,
|
||||
priorities=0,
|
||||
),
|
||||
PoolingEngineInput(
|
||||
prompts=tokens_input(prompt_token_ids=[2]),
|
||||
params=PoolingParams(task="token_embed"),
|
||||
lora_requests=None,
|
||||
priorities=0,
|
||||
),
|
||||
],
|
||||
n_queries=1,
|
||||
)
|
||||
@@ -39,10 +49,13 @@ def _query_key(context: PoolingServeContext) -> str:
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_colliding_request_ids_use_distinct_query_cache_keys():
|
||||
async def test_colliding_request_ids_use_distinct_query_cache_keys(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
serving = object.__new__(ServingScores)
|
||||
serving._prepare_generators = AsyncMock()
|
||||
serving._collect_batch = AsyncMock()
|
||||
prepare_generators = AsyncMock()
|
||||
monkeypatch.setattr(serving, "_prepare_generators", prepare_generators)
|
||||
monkeypatch.setattr(serving, "_collect_batch", AsyncMock())
|
||||
|
||||
first = _make_context()
|
||||
second = _make_context()
|
||||
@@ -50,9 +63,7 @@ async def test_colliding_request_ids_use_distinct_query_cache_keys():
|
||||
await serving._flash_late_interaction_encode_queries(context)
|
||||
await serving._flash_late_interaction_encode_docs(context)
|
||||
|
||||
prepared_contexts = [
|
||||
call.args[0] for call in serving._prepare_generators.await_args_list
|
||||
]
|
||||
prepared_contexts = [call.args[0] for call in prepare_generators.await_args_list]
|
||||
first_query_key = _query_key(prepared_contexts[0])
|
||||
first_doc_key = _query_key(prepared_contexts[1])
|
||||
second_query_key = _query_key(prepared_contexts[2])
|
||||
|
||||
@@ -53,8 +53,8 @@ def test_rejects_mismatched_lora_requests(processor: PoolingIOProcessor):
|
||||
|
||||
|
||||
def test_rejects_conflicting_pooling_task(processor: PoolingIOProcessor):
|
||||
processor.model_config = SimpleNamespace(is_encoder_decoder=False)
|
||||
processor.renderer = SimpleNamespace(
|
||||
processor.model_config = SimpleNamespace(is_encoder_decoder=False) # type: ignore[assignment]
|
||||
processor.renderer = SimpleNamespace( # type: ignore[assignment] # Tokenization defaults only.
|
||||
default_cmpl_tok_params=TokenizeParams(max_total_tokens=None)
|
||||
)
|
||||
ctx = OfflineEncodeInputsContext(
|
||||
|
||||
@@ -45,6 +45,7 @@ async def test_pooling_token_classify(server: RemoteOpenAIServer, model_name: st
|
||||
poolings = PoolingResponse.model_validate(response.json())
|
||||
assert len(poolings.data) == 1
|
||||
assert len(poolings.data[0].data) == 8
|
||||
assert isinstance(poolings.data[0].data[0], list)
|
||||
assert len(poolings.data[0].data[0]) == 2
|
||||
|
||||
|
||||
|
||||
@@ -71,6 +71,7 @@ async def test_pooling_token_embed(server: RemoteOpenAIServer, model_name: str):
|
||||
assert poolings.id == f"pooling-{request_id}"
|
||||
assert len(poolings.data) == 1
|
||||
assert len(poolings.data[0].data) == len(input_tokens)
|
||||
assert isinstance(poolings.data[0].data[0], list)
|
||||
assert len(poolings.data[0].data[0]) == 384
|
||||
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from contextlib import contextmanager
|
||||
from typing import Literal
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
@@ -14,6 +15,7 @@ from vllm.config import (
|
||||
VllmConfig,
|
||||
set_current_vllm_config,
|
||||
)
|
||||
from vllm.config.cache import CacheDType
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.platforms.cpu import CpuPlatform
|
||||
from vllm.platforms.interface import DeviceCapability
|
||||
@@ -21,12 +23,12 @@ from vllm.platforms.interface import DeviceCapability
|
||||
if current_platform.is_cuda():
|
||||
from vllm.platforms.cuda import CudaPlatform
|
||||
else:
|
||||
CudaPlatform = None
|
||||
CudaPlatform = None # type: ignore[assignment] # Unavailable platform import.
|
||||
|
||||
if current_platform.is_rocm():
|
||||
from vllm.platforms.rocm import RocmPlatform
|
||||
else:
|
||||
RocmPlatform = None
|
||||
RocmPlatform = None # type: ignore[misc, assignment] # Unavailable platform import.
|
||||
|
||||
from vllm.v1.attention.backend import AttentionType
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
@@ -354,14 +356,14 @@ def test_invalid_backend():
|
||||
def test_auto_backend_string(auto_value: str):
|
||||
"""Test that 'auto' string value triggers automatic backend selection."""
|
||||
# Using "auto" should result in backend=None (automatic selection)
|
||||
attention_config = AttentionConfig(backend=auto_value)
|
||||
attention_config = AttentionConfig(backend=auto_value) # type: ignore[arg-type]
|
||||
assert attention_config.backend is None
|
||||
|
||||
|
||||
def test_auto_backend_selection_behavior():
|
||||
"""Test that 'auto' backend behaves same as None (automatic selection)."""
|
||||
# Create config with explicit "auto"
|
||||
auto_config = AttentionConfig(backend="auto")
|
||||
auto_config = AttentionConfig(backend="auto") # type: ignore[arg-type]
|
||||
|
||||
# Create config with None (default)
|
||||
none_config = AttentionConfig(backend=None)
|
||||
@@ -406,7 +408,7 @@ def test_auto_backend_selection_behavior():
|
||||
reason="Attention backend FA3 is not supported on ROCm. This test can't succeed.",
|
||||
)
|
||||
def test_per_head_quant_scales_backend_selection(
|
||||
backend_name: str, flash_attn_version: int | None, should_succeed: bool
|
||||
backend_name: str, flash_attn_version: Literal[2, 3, 4] | None, should_succeed: bool
|
||||
):
|
||||
"""Test backend selection when use_per_head_quant_scales=True."""
|
||||
# Clear cache to ensure fresh backend selection
|
||||
@@ -559,7 +561,7 @@ def test_non_causal_autoselect_backend():
|
||||
"int8_per_token_head",
|
||||
],
|
||||
)
|
||||
def test_flash_attn_rejects_unhandled_kv_cache_dtypes(kv_cache_dtype: str):
|
||||
def test_flash_attn_rejects_unhandled_kv_cache_dtypes(kv_cache_dtype: CacheDType):
|
||||
"""FlashAttentionBackend must not claim support for kv_cache dtypes
|
||||
that it cannot handle."""
|
||||
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
|
||||
@@ -569,7 +571,7 @@ def test_flash_attn_rejects_unhandled_kv_cache_dtypes(kv_cache_dtype: str):
|
||||
|
||||
@pytest.mark.parametrize("kv_cache_dtype", ["fp8", "fp8_e4m3"])
|
||||
def test_flash_attn_accepts_handled_fp8_variants(
|
||||
kv_cache_dtype: str, monkeypatch: pytest.MonkeyPatch
|
||||
kv_cache_dtype: CacheDType, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
"""FlashAttentionBackend must accept the two fp8 dtypes it can actually
|
||||
handle: 'fp8' (alias for fp8_e4m3fn) and 'fp8_e4m3'."""
|
||||
@@ -876,7 +878,9 @@ def test_rswa_selection_does_not_reuse_causal_result(blackwell_selection):
|
||||
config = EngineArgs(
|
||||
model="google/gemma-4-31B-it",
|
||||
dtype="bfloat16",
|
||||
attention_config={"backend": "TRITON_FLASHINFER"},
|
||||
attention_config=AttentionConfig(
|
||||
backend=AttentionBackendEnum.TRITON_FLASHINFER
|
||||
),
|
||||
).create_engine_config()
|
||||
with set_current_vllm_config(config):
|
||||
assert (
|
||||
|
||||
@@ -151,7 +151,7 @@ def test_cascade(
|
||||
max_seqlen_k=max_kv_len,
|
||||
softmax_scale=scale,
|
||||
causal=True,
|
||||
window_size=window_size,
|
||||
window_size=list(window_size),
|
||||
block_table=block_tables,
|
||||
softcap=soft_cap if soft_cap is not None else 0,
|
||||
)
|
||||
|
||||
@@ -836,7 +836,7 @@ def varlen_encoder_zentorch_sdpa(
|
||||
key,
|
||||
value,
|
||||
output,
|
||||
SimpleNamespace(query_start_loc=query_start_loc, causal=False),
|
||||
SimpleNamespace(query_start_loc=query_start_loc, causal=False), # type: ignore[arg-type]
|
||||
scale,
|
||||
sliding_window if sliding_window is not None else -1,
|
||||
alibi_slopes,
|
||||
|
||||
@@ -187,7 +187,7 @@ def test_varlen_with_paged_kv(
|
||||
max_seqlen_k=max_kv_len,
|
||||
softmax_scale=scale,
|
||||
causal=True,
|
||||
window_size=window_size,
|
||||
window_size=list(window_size),
|
||||
block_table=block_tables,
|
||||
softcap=soft_cap if soft_cap is not None else 0,
|
||||
fa_version=fa_version,
|
||||
@@ -274,7 +274,7 @@ def test_fa4_hd256_paged_call_shape(
|
||||
max_seqlen_k=num_pages * block_size,
|
||||
softmax_scale=scale,
|
||||
causal=True,
|
||||
window_size=window_size,
|
||||
window_size=list(window_size),
|
||||
block_table=block_tables[:, :num_pages],
|
||||
num_splits=1,
|
||||
fa_version=4,
|
||||
|
||||
@@ -197,11 +197,6 @@ def test_decode_concat_ignores_negative_slots_for_cache(cache_format: str) -> No
|
||||
mixed_cache = initial_cache.clone()
|
||||
reference_cache = _reference_cache(cache_format, inputs, initial_cache, slots)
|
||||
scale_inv = torch.ones(1, device="cuda", dtype=torch.float32)
|
||||
kwargs = {
|
||||
"ds_mla": cache_format == "fp8_ds_mla",
|
||||
"q_scale_inv": scale_inv if cache_format == "fp8" else None,
|
||||
"cache_scale_inv": scale_inv if cache_format == "fp8" else None,
|
||||
}
|
||||
|
||||
output = fused_mla_decode_q_concat_kv_cache_insert(
|
||||
inputs["ql_nope"],
|
||||
@@ -210,7 +205,9 @@ def test_decode_concat_ignores_negative_slots_for_cache(cache_format: str) -> No
|
||||
inputs["k_pe"],
|
||||
mixed_cache,
|
||||
slots,
|
||||
**kwargs,
|
||||
ds_mla=cache_format == "fp8_ds_mla",
|
||||
q_scale_inv=scale_inv if cache_format == "fp8" else None,
|
||||
cache_scale_inv=scale_inv if cache_format == "fp8" else None,
|
||||
)
|
||||
|
||||
expected = _latent_query(inputs["ql_nope"], inputs["q_pe"])
|
||||
|
||||
@@ -277,7 +277,7 @@ def test_mha_attn_varlen_forward_flashinfer(
|
||||
# Override vllm config so get_vit_attn_backend returns FLASHINFER (simulates
|
||||
# --mm-encoder-attn-backend=FLASHINFER).
|
||||
vllm_config = get_current_vllm_config()
|
||||
old_model_config = getattr(vllm_config, "model_config", None)
|
||||
old_model_config = vllm_config.model_config
|
||||
minimal_model_config = type(
|
||||
"MinimalModelConfig",
|
||||
(),
|
||||
@@ -376,7 +376,7 @@ def test_mha_attn_varlen_forward_aiter_fp8(
|
||||
torch.set_default_dtype(dtype)
|
||||
|
||||
vllm_config = get_current_vllm_config()
|
||||
old_model_config = getattr(vllm_config, "model_config", None)
|
||||
old_model_config = vllm_config.model_config
|
||||
minimal_model_config = type(
|
||||
"MinimalModelConfig",
|
||||
(),
|
||||
@@ -442,7 +442,7 @@ def test_mha_attn_varlen_forward_aiter_fp8(
|
||||
)
|
||||
def test_mha_attn_aiter_fp8_rejects_unsupported_arch(default_vllm_config):
|
||||
vllm_config = get_current_vllm_config()
|
||||
old_model_config = getattr(vllm_config, "model_config", None)
|
||||
old_model_config = vllm_config.model_config
|
||||
vllm_config.model_config = type(
|
||||
"MinimalModelConfig",
|
||||
(),
|
||||
@@ -468,7 +468,7 @@ def test_mha_attn_aiter_fp8_rejects_unsupported_arch(default_vllm_config):
|
||||
)
|
||||
def test_mha_attn_fp8_rejects_wrong_backend(default_vllm_config):
|
||||
vllm_config = get_current_vllm_config()
|
||||
old_model_config = getattr(vllm_config, "model_config", None)
|
||||
old_model_config = vllm_config.model_config
|
||||
vllm_config.model_config = type(
|
||||
"MinimalModelConfig",
|
||||
(),
|
||||
|
||||
@@ -1689,8 +1689,10 @@ def test_aiter_sparse_pa_layout_contract(monkeypatch, layout):
|
||||
)
|
||||
|
||||
backend = sparse_attn_mod.MiniMaxM3SparseBackend
|
||||
assert KVCacheLayout.LHBNC in backend.supported_kv_cache_layouts()
|
||||
assert KVCacheLayout.LBHNC in backend.supported_kv_cache_layouts()
|
||||
layouts = backend.supported_kv_cache_layouts()
|
||||
assert layouts is not None
|
||||
assert KVCacheLayout.LHBNC in layouts
|
||||
assert KVCacheLayout.LBHNC in layouts
|
||||
|
||||
nb, h = 3, 1
|
||||
cache_dtype = current_platform.fp8_dtype()
|
||||
@@ -1772,19 +1774,22 @@ def test_aiter_indexer_requires_the_aiter_attend(monkeypatch):
|
||||
import vllm.models.minimax_m3.amd.indexer_aiter as indexer_aiter_mod
|
||||
|
||||
monkeypatch.setattr(indexer_aiter_mod.current_platform, "is_rocm", lambda: True)
|
||||
kwargs = dict(
|
||||
topk_blocks=TOPK,
|
||||
sparse_block_size=BLOCK_SIZE,
|
||||
num_index_heads=1,
|
||||
index_head_dim=HEAD_DIM,
|
||||
indexer_kv_dtype="fp8_e4m3",
|
||||
max_model_len=8192,
|
||||
)
|
||||
|
||||
def unsupported_reason() -> str | None:
|
||||
return indexer_aiter_mod.aiter_indexer_unsupported_reason(
|
||||
topk_blocks=TOPK,
|
||||
sparse_block_size=BLOCK_SIZE,
|
||||
num_index_heads=1,
|
||||
index_head_dim=HEAD_DIM,
|
||||
# The implementation also accepts this legacy fp8 alias.
|
||||
indexer_kv_dtype="fp8_e4m3", # type: ignore[arg-type]
|
||||
max_model_len=8192,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
indexer_aiter_mod, "_minimax_m3_aiter_sparse_pa_requested", lambda: False
|
||||
)
|
||||
reason = indexer_aiter_mod.aiter_indexer_unsupported_reason(**kwargs)
|
||||
reason = unsupported_reason()
|
||||
assert reason is not None and "AITER sparse PA attend" in reason
|
||||
|
||||
# With the attend asked for, the gate moves on to the kernel-contract
|
||||
@@ -1792,7 +1797,7 @@ def test_aiter_indexer_requires_the_aiter_attend(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
indexer_aiter_mod, "_minimax_m3_aiter_sparse_pa_requested", lambda: True
|
||||
)
|
||||
reason = indexer_aiter_mod.aiter_indexer_unsupported_reason(**kwargs)
|
||||
reason = unsupported_reason()
|
||||
assert reason is None or "AITER sparse PA attend" not in reason
|
||||
|
||||
|
||||
|
||||
@@ -235,14 +235,14 @@ def test_msa_metadata_builder_prepares_cutlass_for_regular_decode(
|
||||
builder.num_q_heads = 64
|
||||
builder.num_kv_heads = 4
|
||||
builder.topk_blocks = TOPK
|
||||
builder.kv_cache_spec = SimpleNamespace(num_kv_heads=4)
|
||||
builder.kv_cache_spec = SimpleNamespace(num_kv_heads=4) # type: ignore[assignment]
|
||||
builder.kv_cache_dtype = "fp8_e4m3"
|
||||
builder.decode_backend = "cutlass"
|
||||
builder.msa_cutlass_plan_cache = object()
|
||||
builder.msa_cutlass_plan_cache = object() # type: ignore[assignment] # Mocked plan-cache sentinel.
|
||||
|
||||
metadata = builder.build(
|
||||
0,
|
||||
SimpleNamespace(
|
||||
SimpleNamespace( # type: ignore[arg-type]
|
||||
seq_lens_cpu_upper_bound=torch.full((batch,), 257, dtype=torch.int32)
|
||||
),
|
||||
)
|
||||
@@ -377,7 +377,7 @@ def test_query_fp8_stays_valid_when_cutlass_plan_appears_on_replay(
|
||||
with torch.cuda.stream(stream), override_forward_context(forward_context):
|
||||
capture = BreakableCUDAGraphCapture()
|
||||
with capture:
|
||||
query_fp8 = MiniMaxM3SparseAttention._allocate_query_fp8(attention, qkv)
|
||||
query_fp8 = MiniMaxM3SparseAttention._allocate_query_fp8(attention, qkv) # type: ignore[arg-type]
|
||||
if query_fp8 is not None:
|
||||
query_fp8.zero_()
|
||||
run_attention(query_fp8)
|
||||
|
||||
@@ -783,6 +783,9 @@ def test_aiter_fa_shared_kv_matches_reference(
|
||||
32 * BLOCK_SIZE, num_kv_heads, head_size, dtype=torch.bfloat16
|
||||
)
|
||||
value = torch.randn_like(key)
|
||||
from vllm.v1.attention.backends.rocm_aiter_fa import AiterFlashAttentionImpl
|
||||
|
||||
assert isinstance(target.impl, AiterFlashAttentionImpl)
|
||||
target.impl.do_kv_cache_update(
|
||||
target,
|
||||
key,
|
||||
|
||||
@@ -82,6 +82,7 @@ def _run_verify_block():
|
||||
)
|
||||
from vllm.config import SpeculativeConfig
|
||||
from vllm.config.vllm import set_current_vllm_config
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAImpl
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.kv_cache_interface import MLAAttentionSpec
|
||||
from vllm.v1.worker.workspace import init_workspace_manager
|
||||
@@ -106,7 +107,7 @@ def _run_verify_block():
|
||||
vllm_config.speculative_config = SpeculativeConfig(
|
||||
method="ngram", num_speculative_tokens=QLEN - 1
|
||||
)
|
||||
vllm_config.model_config.get_num_attention_heads = types.MethodType(
|
||||
vllm_config.model_config.get_num_attention_heads = types.MethodType( # type: ignore[method-assign]
|
||||
lambda self, parallel_config, arch_config=None: NUM_QUERY_HEADS,
|
||||
vllm_config.model_config,
|
||||
)
|
||||
@@ -202,7 +203,8 @@ def _run_verify_block():
|
||||
lambda: spy,
|
||||
),
|
||||
):
|
||||
impl.forward_mqa((q_nope, q_pe), kv_cache, metadata, layer=None)
|
||||
assert isinstance(impl, AiterMLAImpl)
|
||||
impl.forward_mqa((q_nope, q_pe), kv_cache, metadata, layer=None) # type: ignore[arg-type] # Spy decode path does not read layer.
|
||||
|
||||
return metadata, captured
|
||||
|
||||
|
||||
@@ -121,7 +121,7 @@ def _build_prefill_metadata(
|
||||
|
||||
builder = object.__new__(AiterMLAMetadataBuilder)
|
||||
builder.num_heads = num_heads
|
||||
builder.mla_dims = SimpleNamespace(v_head_dim=V_HEAD_DIM)
|
||||
builder.mla_dims = SimpleNamespace(v_head_dim=V_HEAD_DIM) # type: ignore[assignment]
|
||||
builder._init_fp8_prefill_ps_buffers(
|
||||
max_num_reqs=len(seq_lens),
|
||||
max_prefill_qlen=max_q,
|
||||
@@ -136,7 +136,7 @@ def _build_prefill_metadata(
|
||||
prefill = SimpleNamespace(query_start_loc=qo_indptr, max_query_len=max_q)
|
||||
metadata = SimpleNamespace(prefill=prefill, num_decodes=0)
|
||||
common = SimpleNamespace(query_start_loc_cpu=qo_indptr_cpu)
|
||||
builder._build_fp8_prefill_ps_metadata(metadata, common)
|
||||
builder._build_fp8_prefill_ps_metadata(metadata, common) # type: ignore[arg-type]
|
||||
return metadata, total_q, builder
|
||||
|
||||
|
||||
|
||||
@@ -167,7 +167,10 @@ def test_sparse_mla_sink_matches_ragged_reference(
|
||||
impl.sinks = sinks
|
||||
|
||||
output, lse = impl._forward_mla(
|
||||
SimpleNamespace(_q_scale=q_scale, _k_scale=kv_scale), q, kv, metadata
|
||||
SimpleNamespace(_q_scale=q_scale, _k_scale=kv_scale),
|
||||
q,
|
||||
kv,
|
||||
metadata, # type: ignore[arg-type]
|
||||
)
|
||||
kv_flat = kv_ref[:, 0]
|
||||
references = []
|
||||
@@ -241,7 +244,7 @@ def test_sparse_mla_sink_rejects_unsupported_aiter_dtypes(
|
||||
SimpleNamespace(_q_scale=None, _k_scale=None),
|
||||
torch.empty(1, 16, Q_HEAD_DIM, dtype=q_dtype),
|
||||
torch.empty(1, 1, Q_HEAD_DIM, dtype=kv_dtype),
|
||||
metadata,
|
||||
metadata, # type: ignore[arg-type]
|
||||
)
|
||||
|
||||
|
||||
@@ -381,7 +384,7 @@ def test_sparse_mla_sink_matches_dense_attention_with_empty_rows_and_paged_cache
|
||||
paged_kv_indices=torch.tensor(indices, dtype=torch.int32, device="cuda"),
|
||||
paged_kv_indptr=lengths.cumsum(0).to(device="cuda", dtype=torch.int32),
|
||||
)
|
||||
impl = ROCMAiterMLASparseImpl.__new__(ROCMAiterMLASparseImpl)
|
||||
impl = ROCMAiterMLASparseImpl.__new__(ROCMAiterMLASparseImpl) # type: ignore[arg-type]
|
||||
impl.num_heads = num_heads
|
||||
impl.head_size = head_dim
|
||||
impl.kv_lora_rank = value_dim
|
||||
@@ -438,9 +441,9 @@ def test_sparse_mla_backend_resolves_only_contiguous_layer_layouts(monkeypatch,
|
||||
]
|
||||
if layout == "BLNHC":
|
||||
with pytest.raises(ValueError, match="does not satisfy"):
|
||||
resolve_kv_cache_layout(config, supported)
|
||||
resolve_kv_cache_layout(config, supported) # type: ignore[arg-type]
|
||||
else:
|
||||
assert resolve_kv_cache_layout(config, supported).name == layout
|
||||
assert resolve_kv_cache_layout(config, supported).name == layout # type: ignore[arg-type]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
||||
@@ -491,7 +494,7 @@ def test_sparse_mla_sink_forward_mqa_preserves_split_query(dtype):
|
||||
actual, _ = impl.forward_mqa(
|
||||
(q[..., :V_HEAD_DIM], q[..., V_HEAD_DIM:]),
|
||||
kv,
|
||||
metadata,
|
||||
metadata, # type: ignore[arg-type]
|
||||
SimpleNamespace(_q_scale=None, _k_scale=None),
|
||||
)
|
||||
references = [
|
||||
|
||||
@@ -39,10 +39,10 @@ def _make_builder():
|
||||
topk_tokens = 4
|
||||
|
||||
builder.device = torch.device("cpu")
|
||||
builder.kv_cache_spec = SimpleNamespace(block_size=1)
|
||||
builder.kv_cache_spec = SimpleNamespace(block_size=1) # type: ignore[assignment]
|
||||
builder.model_dtype = torch.bfloat16
|
||||
builder.kv_cache_dtype = "fp8"
|
||||
builder.mla_dims = SimpleNamespace(kv_lora_rank=512, qk_rope_head_dim=64)
|
||||
builder.mla_dims = SimpleNamespace(kv_lora_rank=512, qk_rope_head_dim=64) # type: ignore[assignment]
|
||||
builder.topk_tokens = topk_tokens
|
||||
builder.req_id_per_token_buffer = torch.zeros(
|
||||
max_num_batched_tokens, dtype=torch.int32, device="cpu"
|
||||
|
||||
@@ -227,7 +227,9 @@ def _decode_metadata(case, rows, ratio, query_lens):
|
||||
num_decode_tokens=len(rows),
|
||||
num_prefills=0,
|
||||
num_prefill_tokens=0,
|
||||
decode=types.SimpleNamespace(block_table=block_table, seq_lens=lens[:, None]),
|
||||
decode=types.SimpleNamespace( # type: ignore[arg-type]
|
||||
block_table=block_table, seq_lens=lens[:, None]
|
||||
),
|
||||
decode_row_lens=lens,
|
||||
decode_block_ends=(lens + CAND_BLOCK - 1) // CAND_BLOCK,
|
||||
decode_native=native,
|
||||
@@ -278,7 +280,7 @@ def _prefill_metadata(
|
||||
num_decode_tokens=0,
|
||||
num_prefills=len(case.seq_lens),
|
||||
num_prefill_tokens=len(rows),
|
||||
prefill=types.SimpleNamespace(chunks=chunks),
|
||||
prefill=types.SimpleNamespace(chunks=chunks), # type: ignore[arg-type]
|
||||
prefill_plans=plans,
|
||||
gather_launches=plan_gather_launches(chunks, plans, gather_rows),
|
||||
)
|
||||
|
||||
@@ -955,7 +955,7 @@ def test_rocm_capture_metadata_sets_adaptive_marker(monkeypatch) -> None:
|
||||
)
|
||||
builder = object.__new__(rocm_mod.DeepseekV4ROCMAiterMLASparseMetadataBuilder)
|
||||
|
||||
actual = builder.build_for_cudagraph_capture(SimpleNamespace())
|
||||
actual = builder.build_for_cudagraph_capture(SimpleNamespace()) # type: ignore[arg-type] # Patched base method ignores metadata.
|
||||
|
||||
assert actual is metadata
|
||||
assert actual.for_cudagraph_capture is _on_gfx950()
|
||||
@@ -1563,7 +1563,7 @@ def test_dsv4_adaptive_mla_swa_metadata_graph_replay(monkeypatch) -> None:
|
||||
"sliding_window": window_size,
|
||||
},
|
||||
)
|
||||
vllm_config.speculative_config = SimpleNamespace(
|
||||
vllm_config.speculative_config = SimpleNamespace( # type: ignore[assignment]
|
||||
num_speculative_tokens=upper_query_len - 1,
|
||||
parallel_drafting=False,
|
||||
enable_adaptive_verification=True,
|
||||
|
||||
@@ -153,6 +153,7 @@ def test_triton_unified_attn_diffkv_vs_reference(
|
||||
else:
|
||||
from vllm.v1.attention.backends.fa_utils import flash_attn_varlen_func
|
||||
|
||||
assert fa_version is not None
|
||||
# FA's TMA path needs aligned singleton strides (num_kv_heads == 1).
|
||||
fa_k = canonicalize_singleton_dim_strides(key_cache)
|
||||
fa_v = canonicalize_singleton_dim_strides(value_cache)
|
||||
|
||||
@@ -79,7 +79,7 @@ class TestConfigSet:
|
||||
config_set = ConfigSet("test_kernel")
|
||||
|
||||
with pytest.raises(KeyError, match="platform 'h100' not found"):
|
||||
config_set.get_config("h100", "nonexistent")
|
||||
config_set.get_config("h100", CaseKey({"batch": 32}))
|
||||
|
||||
config_data = {"num_warps": 8, "num_stages": 4}
|
||||
data = {
|
||||
|
||||
@@ -81,7 +81,7 @@ class TestDynamicPerTokenScaledFp8QuantConfigPicker:
|
||||
assert selected_key == CaseKey({"hidden_size": 2048, "num_tokens": 32})
|
||||
|
||||
def test_config_picker_no_configs(self):
|
||||
config_keys: list[dict] = []
|
||||
config_keys: list[CaseKey] = []
|
||||
|
||||
args = _generate_fake_input(16, 4096)
|
||||
selected_key = pick_config(args, config_keys)
|
||||
|
||||
@@ -127,7 +127,7 @@ class TestFusedQkNormRopeConfigPicker:
|
||||
)
|
||||
|
||||
def test_config_picker_no_configs(self):
|
||||
config_keys: list[dict] = []
|
||||
config_keys: list[CaseKey] = []
|
||||
|
||||
args = _generate_fake_input(16, 4096, 128)
|
||||
selected_key = pick_config(args, config_keys)
|
||||
|
||||
@@ -38,6 +38,7 @@ from torch._inductor.pattern_matcher import (
|
||||
)
|
||||
from torch.fx.experimental.proxy_tensor import make_fx
|
||||
|
||||
from vllm.kernels.helion.case_key import CaseKey
|
||||
from vllm.kernels.helion.config_manager import ConfigManager
|
||||
from vllm.kernels.helion.register import HelionKernelWrapper
|
||||
|
||||
@@ -45,7 +46,7 @@ from vllm.kernels.helion.register import HelionKernelWrapper
|
||||
@contextlib.contextmanager
|
||||
def _helion_mock_context():
|
||||
configs = {
|
||||
"default": helion.Config(block_sizes=[64], num_warps=2, num_stages=2),
|
||||
CaseKey.default(): helion.Config(block_sizes=[64], num_warps=2, num_stages=2),
|
||||
}
|
||||
mock_config_manager = Mock(spec=ConfigManager)
|
||||
mock_config_manager.get_platform_configs = Mock(return_value=configs)
|
||||
@@ -88,7 +89,7 @@ class TestMakeFxHop:
|
||||
raw_kernel_func=raw_add_scale,
|
||||
op_name="test_make_fx",
|
||||
fake_impl=lambda *a, **kw: None,
|
||||
config_picker=lambda args, keys: "default",
|
||||
config_picker=lambda args, keys: CaseKey.default(),
|
||||
)
|
||||
|
||||
def fn(x, y):
|
||||
@@ -145,7 +146,7 @@ class TestMakeFxHop:
|
||||
raw_kernel_func=raw_silu_mul,
|
||||
op_name="test_pm_silu_mul",
|
||||
fake_impl=lambda *a, **kw: None,
|
||||
config_picker=lambda args, keys: "default",
|
||||
config_picker=lambda args, keys: CaseKey.default(),
|
||||
)
|
||||
|
||||
def pattern(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
|
||||
|
||||
@@ -107,7 +107,7 @@ class TestPerTokenGroupFp8QuantConfigPicker:
|
||||
)
|
||||
|
||||
def test_config_picker_no_configs(self):
|
||||
config_keys: list[dict] = []
|
||||
config_keys: list[CaseKey] = []
|
||||
|
||||
args = _generate_fake_input(16, 4096, 128)
|
||||
selected_key = pick_config(args, config_keys)
|
||||
|
||||
@@ -397,6 +397,7 @@ class TestHelionKernelWrapper:
|
||||
)
|
||||
|
||||
assert wrapper._disabled is True
|
||||
assert wrapper._disabled_reason is not None
|
||||
assert "No configs available" in wrapper._disabled_reason
|
||||
|
||||
def test_disabled_wrapper_raises_on_call(self, sample_kernel):
|
||||
|
||||
@@ -89,7 +89,7 @@ class TestRmsNormDynamicPerTokenQuantConfigPicker:
|
||||
assert selected_key == CaseKey({"hidden_size": 2048, "num_tokens": 32})
|
||||
|
||||
def test_config_picker_no_configs(self):
|
||||
config_keys: list[dict] = []
|
||||
config_keys: list[CaseKey] = []
|
||||
|
||||
args = _generate_fake_input(16, 4096)
|
||||
selected_key = pick_config(args, config_keys)
|
||||
|
||||
@@ -112,7 +112,7 @@ class TestRmsNormPerBlockQuantConfigPicker:
|
||||
)
|
||||
|
||||
def test_config_picker_no_configs(self):
|
||||
config_keys: list[dict] = []
|
||||
config_keys: list[CaseKey] = []
|
||||
|
||||
args = _generate_fake_input(16, 4096, 128)
|
||||
selected_key = pick_config(args, config_keys)
|
||||
|
||||
@@ -96,7 +96,7 @@ class TestSiluAndMulPerBlockQuantConfigPicker:
|
||||
)
|
||||
|
||||
def test_config_picker_no_configs(self):
|
||||
config_keys: list[dict] = []
|
||||
config_keys: list[CaseKey] = []
|
||||
|
||||
args = _generate_fake_input(16, 4096, 128)
|
||||
selected_key = pick_config(args, config_keys)
|
||||
|
||||
@@ -83,7 +83,7 @@ class TestSiluMulFp8ConfigPicker:
|
||||
assert selected_key == CaseKey({"intermediate": 4096, "numtokens": 256})
|
||||
|
||||
def test_config_picker_no_configs(self):
|
||||
config_keys: list[dict] = []
|
||||
config_keys: list[CaseKey] = []
|
||||
|
||||
input_tensor = torch.randn(32, 4096, dtype=torch.bfloat16, device="cuda")
|
||||
scale = torch.tensor([0.5], dtype=torch.float32, device="cuda")
|
||||
@@ -355,6 +355,7 @@ class TestSiluMulFp8Integration:
|
||||
registered_kernels = get_registered_kernels()
|
||||
kernel_wrapper = registered_kernels["silu_mul_fp8"]
|
||||
fake_impl = kernel_wrapper._fake_impl
|
||||
assert fake_impl is not None
|
||||
|
||||
fake_output = fake_impl(input_tensor, scale)
|
||||
|
||||
|
||||
@@ -7,6 +7,15 @@ import pytest
|
||||
|
||||
from tests.conftest import HfRunner
|
||||
from tests.models.utils import EmbedModelInfo, check_embeddings_close, matryoshka_fy
|
||||
from vllm.entrypoints.pooling.embed.protocol import EmbeddingResponse
|
||||
|
||||
|
||||
def float_embeddings(response: EmbeddingResponse) -> list[list[float]]:
|
||||
embeddings = []
|
||||
for d in response.data:
|
||||
assert isinstance(d.embedding, list)
|
||||
embeddings.append(d.embedding)
|
||||
return embeddings
|
||||
|
||||
|
||||
def run_embedding_correctness_test(
|
||||
|
||||
@@ -53,12 +53,10 @@ def test_video_media_io_kwargs(monkeypatch: pytest.MonkeyPatch):
|
||||
imageio = ImageMediaIO()
|
||||
|
||||
# Verify that different args pass/fail assertions as expected.
|
||||
videoio = VideoMediaIO(imageio, **{"num_frames": 10, "fps": 1.0})
|
||||
videoio = VideoMediaIO(imageio, num_frames=10, fps=1.0)
|
||||
_ = videoio.load_bytes(b"test")
|
||||
|
||||
videoio = VideoMediaIO(
|
||||
imageio, **{"num_frames": 10, "fps": 1.0, "not_used": "not_used"}
|
||||
)
|
||||
videoio = VideoMediaIO(imageio, num_frames=10, fps=1.0, not_used="not_used")
|
||||
_ = videoio.load_bytes(b"test")
|
||||
|
||||
with pytest.raises(AssertionError, match="bad num_frames"):
|
||||
@@ -66,11 +64,11 @@ def test_video_media_io_kwargs(monkeypatch: pytest.MonkeyPatch):
|
||||
_ = videoio.load_bytes(b"test")
|
||||
|
||||
with pytest.raises(AssertionError, match="bad num_frames"):
|
||||
videoio = VideoMediaIO(imageio, **{"num_frames": 9, "fps": 1.0})
|
||||
videoio = VideoMediaIO(imageio, num_frames=9, fps=1.0)
|
||||
_ = videoio.load_bytes(b"test")
|
||||
|
||||
with pytest.raises(AssertionError, match="bad fps"):
|
||||
videoio = VideoMediaIO(imageio, **{"num_frames": 10, "fps": 2.0})
|
||||
videoio = VideoMediaIO(imageio, num_frames=10, fps=2.0)
|
||||
_ = videoio.load_bytes(b"test")
|
||||
|
||||
|
||||
@@ -86,14 +84,14 @@ def test_opencv_video_io_colorspace(tmp_path, is_color: bool, fourcc: str, ext:
|
||||
image = Image.open(image_path)
|
||||
|
||||
if not is_color:
|
||||
image_path = f"{tmp_path}/test_grayscale_image.png"
|
||||
image_path = Path(tmp_path) / "test_grayscale_image.png"
|
||||
image = image.convert("L")
|
||||
image.save(image_path)
|
||||
# Convert to gray RGB for comparison
|
||||
image = image.convert("RGB")
|
||||
video_path = f"{tmp_path}/test_RGB_video.{ext}"
|
||||
create_video_from_image(
|
||||
image_path,
|
||||
str(image_path),
|
||||
video_path,
|
||||
num_frames=2,
|
||||
is_color=is_color,
|
||||
@@ -385,7 +383,9 @@ def test_pynvvideocodec_unrelated_error_propagates(
|
||||
|
||||
with pytest.raises(RuntimeError) as exc_info:
|
||||
PyNvVideoCodecVideoBackendMixin.decode_frames_pynvvideocodec(
|
||||
None, b"video", None
|
||||
None,
|
||||
b"video",
|
||||
None, # type: ignore[arg-type] # Fails before target metadata is used.
|
||||
)
|
||||
|
||||
assert exc_info.value is original_error
|
||||
|
||||
@@ -179,7 +179,7 @@ def test_audio_resampler_invalid_method(dummy_audio):
|
||||
# Validated eagerly so a bad method fails at construction, not on the
|
||||
# first audio request.
|
||||
with pytest.raises(ValueError, match="Invalid resampling method"):
|
||||
AudioResampler(target_sr=22050, method="invalid")
|
||||
AudioResampler(target_sr=22050, method="invalid") # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_audio_resampler_no_target_sr(dummy_audio):
|
||||
@@ -378,6 +378,7 @@ class TestMultiModalDataParserChannelNormalization:
|
||||
result = parser._parse_audio_data((stereo_audio, 16000))
|
||||
|
||||
# Check that result is mono (1D)
|
||||
assert result is not None
|
||||
audio_item = result.get(0)
|
||||
assert audio_item.ndim == 1, f"Expected 1D mono audio, got {audio_item.ndim}D"
|
||||
assert audio_item.shape == (3,), f"Expected shape (3,), got {audio_item.shape}"
|
||||
@@ -404,6 +405,7 @@ class TestMultiModalDataParserChannelNormalization:
|
||||
result = parser._parse_audio_data((stereo_audio, 16000))
|
||||
|
||||
# Check that result preserves original shape (after resampling)
|
||||
assert result is not None
|
||||
audio_item = result.get(0)
|
||||
# When target_channels=None, stereo audio should be preserved
|
||||
assert audio_item.ndim == 2, f"Expected 2D stereo audio, got {audio_item.ndim}D"
|
||||
@@ -425,6 +427,7 @@ class TestMultiModalDataParserChannelNormalization:
|
||||
result = parser._parse_audio_data((mono_audio, 16000))
|
||||
|
||||
# Check that result is still mono (1D)
|
||||
assert result is not None
|
||||
audio_item = result.get(0)
|
||||
assert audio_item.ndim == 1
|
||||
assert audio_item.shape == (16000,)
|
||||
@@ -446,6 +449,7 @@ class TestMultiModalDataParserChannelNormalization:
|
||||
result = parser._parse_audio_data((surround_audio, 16000))
|
||||
|
||||
# Check that result is stereo (2 channels)
|
||||
assert result is not None
|
||||
audio_item = result.get(0)
|
||||
assert audio_item.ndim == 2
|
||||
assert audio_item.shape[0] == 2 # 2 channels
|
||||
@@ -483,6 +487,7 @@ class TestAudioPipelineE2E:
|
||||
|
||||
# Process audio through the parser
|
||||
result = parser._parse_audio_data((stereo_torchaudio, 16000))
|
||||
assert result is not None
|
||||
audio_output = result.get(0)
|
||||
|
||||
# Verify output is mono 1D
|
||||
@@ -512,6 +517,7 @@ class TestAudioPipelineE2E:
|
||||
|
||||
# Process audio through the parser
|
||||
result = parser._parse_audio_data((stereo_soundfile, 16000))
|
||||
assert result is not None
|
||||
audio_output = result.get(0)
|
||||
|
||||
# Verify output is mono 1D
|
||||
@@ -537,6 +543,7 @@ class TestAudioPipelineE2E:
|
||||
|
||||
# Process audio through the parser
|
||||
result = parser._parse_audio_data((mono_pyav, 16000))
|
||||
assert result is not None
|
||||
audio_output = result.get(0)
|
||||
|
||||
# Verify output is still mono 1D
|
||||
@@ -572,6 +579,7 @@ class TestAudioPipelineE2E:
|
||||
|
||||
# Process audio through the parser
|
||||
result = parser._parse_audio_data((surround_audio, 16000))
|
||||
assert result is not None
|
||||
audio_output = result.get(0)
|
||||
|
||||
# Verify output is mono 1D
|
||||
@@ -603,6 +611,7 @@ class TestAudioPipelineE2E:
|
||||
# Process audio through the parser
|
||||
# Note: Parser expects numpy, so we convert first (simulating real usage)
|
||||
result = parser._parse_audio_data((stereo_torch.numpy(), 16000))
|
||||
assert result is not None
|
||||
audio_output = result.get(0)
|
||||
|
||||
# Verify output is mono 1D numpy array
|
||||
@@ -630,6 +639,7 @@ class TestAudioPipelineE2E:
|
||||
|
||||
# Process audio through the parser
|
||||
result = parser._parse_audio_data((stereo_audio, 16000))
|
||||
assert result is not None
|
||||
audio_output = result.get(0)
|
||||
|
||||
# Verify output preserves stereo (2D)
|
||||
@@ -654,6 +664,7 @@ class TestAudioPipelineE2E:
|
||||
|
||||
# Process audio through the parser
|
||||
result = parser._parse_audio_data((stereo_48k, 48000))
|
||||
assert result is not None
|
||||
audio_output = result.get(0)
|
||||
|
||||
# Verify output is mono 1D at target sample rate
|
||||
@@ -677,6 +688,7 @@ class TestAudioPipelineE2E:
|
||||
)
|
||||
|
||||
result = parser._parse_audio_data((short_stereo, 16000))
|
||||
assert result is not None
|
||||
audio_output = result.get(0)
|
||||
|
||||
# Should still produce mono output
|
||||
|
||||
@@ -11,7 +11,6 @@ import torch
|
||||
from vllm.config import ModelConfig, ParallelConfig, VllmConfig
|
||||
from vllm.config.multimodal import MultiModalConfig
|
||||
from vllm.multimodal.cache import (
|
||||
BaseMultiModalProcessorCache,
|
||||
BaseMultiModalReceiverCache,
|
||||
LruKeyReplicatedReceiverCache,
|
||||
LruKeyReplicatedSenderCache,
|
||||
@@ -145,6 +144,8 @@ def _compare_caches(
|
||||
cache_1_p0 = processor_cache_from_config(config_1)
|
||||
cache_1_p1 = engine_receiver_cache_from_config(config_1)
|
||||
|
||||
assert config_0.model_config.multimodal_config is not None
|
||||
assert config_1.model_config.multimodal_config is not None
|
||||
cache_size_gb = max(
|
||||
config_0.model_config.multimodal_config.mm_processor_cache_gb,
|
||||
config_1.model_config.multimodal_config.mm_processor_cache_gb,
|
||||
@@ -280,6 +281,7 @@ def test_oversized_item_is_served_uncached():
|
||||
|
||||
p0 = LruKeyReplicatedSenderCache(model_config) # type: ignore[arg-type]
|
||||
p1 = LruKeyReplicatedReceiverCache(model_config) # type: ignore[arg-type]
|
||||
assert item is not None
|
||||
assert p0.get_and_update_item((item, []), "big")[0] is item
|
||||
assert not p0.is_cached_item("big")
|
||||
assert p1.get_and_update_item(item, "big") is item
|
||||
@@ -318,6 +320,7 @@ def test_mm_cache_miss_raises_and_recovers():
|
||||
assert p0.is_cached_item(mm_hash)
|
||||
# On the next request P0 short-circuits to data=None -- the drift bug when
|
||||
# P1 lacks the item.
|
||||
assert item is not None
|
||||
hit = p0.get_and_update_item((item, []), mm_hash)
|
||||
assert hit[0] is None
|
||||
|
||||
@@ -326,6 +329,7 @@ def test_mm_cache_miss_raises_and_recovers():
|
||||
p0.invalidate(mm_hash)
|
||||
assert not p0.is_cached_item(mm_hash)
|
||||
|
||||
assert item is not None
|
||||
resent = p0.get_and_update_item((item, []), mm_hash)
|
||||
assert resent[0] is item # MISS again -> data is resent
|
||||
assert p1.get_and_update_item(item, mm_hash) == item # P1 now caches it
|
||||
@@ -459,6 +463,7 @@ def test_shm_receiver_handles_prefix_covered_items(monkeypatch):
|
||||
address_item, _ = p0.get_and_update_item((item, []), mm_hash)
|
||||
first = _feature(mm_hash, address_item)
|
||||
p1.get_and_update_features([first])
|
||||
assert first.data is not None
|
||||
assert torch.equal(first.data["dummy"].data, item["dummy"].data)
|
||||
|
||||
# Request 2 (identical, fully prefix-covered): the sender hit takes
|
||||
@@ -476,6 +481,7 @@ def test_shm_receiver_handles_prefix_covered_items(monkeypatch):
|
||||
|
||||
assert covered_uncached.data is None
|
||||
# The address item is resolved to the cached payload.
|
||||
assert covered_cached.data is not None
|
||||
assert torch.equal(covered_cached.data["dummy"].data, item["dummy"].data)
|
||||
|
||||
# The hit's writer references were acknowledged by the worker, so the
|
||||
@@ -488,7 +494,7 @@ def test_shm_receiver_handles_prefix_covered_items(monkeypatch):
|
||||
|
||||
|
||||
def _run_test_cache_eviction_lru(
|
||||
p0_cache: BaseMultiModalProcessorCache,
|
||||
p0_cache: LruKeyReplicatedSenderCache,
|
||||
p1_cache: BaseMultiModalReceiverCache,
|
||||
base_item_size: int,
|
||||
):
|
||||
@@ -603,7 +609,7 @@ def test_cache_eviction_lru_cache():
|
||||
# image_B is protected from eviction then image_i cannot be added.
|
||||
# This proving normal eviction and reuse behavior.
|
||||
def _run_test_cache_eviction_shm(
|
||||
p0_cache: BaseMultiModalProcessorCache,
|
||||
p0_cache: ShmObjectStoreSenderCache,
|
||||
p1_cache: BaseMultiModalReceiverCache,
|
||||
base_item_size: int,
|
||||
):
|
||||
@@ -860,9 +866,9 @@ async def test_release_kv_cache_resends_mm_payload(use_async, release_error):
|
||||
|
||||
async def call_release():
|
||||
if use_async:
|
||||
await AsyncLLM.release_kv_cache_memory(engine)
|
||||
await AsyncLLM.release_kv_cache_memory(engine) # type: ignore[arg-type]
|
||||
else:
|
||||
LLMEngine.release_kv_cache_memory(engine)
|
||||
LLMEngine.release_kv_cache_memory(engine) # type: ignore[arg-type]
|
||||
|
||||
if release_error:
|
||||
with pytest.raises(RuntimeError, match=release_error):
|
||||
@@ -894,9 +900,10 @@ def test_sleep_wake_preserves_mm_cache_consistency():
|
||||
"""Regression for vllm-project/vllm#42995."""
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.assets.image import ImageAsset
|
||||
from vllm.inputs import TextPrompt
|
||||
|
||||
image = ImageAsset("stop_sign").pil_image
|
||||
prompt = {
|
||||
prompt: TextPrompt = {
|
||||
"prompt": _SLEEP_VISION_PROMPT,
|
||||
"multi_modal_data": {"image": image},
|
||||
}
|
||||
|
||||
@@ -131,6 +131,7 @@ def test_hash_collision_video_num_frames():
|
||||
}
|
||||
video = MediaWithBytes((frames, metadata), source)
|
||||
items = MultiModalDataParser()._parse_video_data([video])
|
||||
assert items is not None
|
||||
return items.get_all_items_for_hash()[0]
|
||||
|
||||
hasher = MultiModalHasher
|
||||
@@ -155,6 +156,7 @@ def test_hash_video_tensor_frames():
|
||||
}
|
||||
video = MediaWithBytes((frames, metadata), source)
|
||||
items = MultiModalDataParser()._parse_video_data([video])
|
||||
assert items is not None
|
||||
return items.get_all_items_for_hash()[0]
|
||||
|
||||
np_frames = np.zeros((2, 8, 8, 3), dtype=np.uint8)
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import time
|
||||
from collections.abc import Sequence
|
||||
from contextlib import nullcontext
|
||||
from copy import deepcopy
|
||||
from types import SimpleNamespace
|
||||
@@ -868,7 +869,7 @@ def test_limit_mm_per_prompt_apply(model_id, num_images, limit, is_valid):
|
||||
],
|
||||
)
|
||||
def test_budget_caps_prevent_dummy_input_validation_failure(
|
||||
model_id, user_limit, supported_limit
|
||||
model_id, user_limit, supported_limit, monkeypatch
|
||||
):
|
||||
limit_mm_per_prompt = {"image": user_limit}
|
||||
|
||||
@@ -878,7 +879,9 @@ def test_budget_caps_prevent_dummy_input_validation_failure(
|
||||
)
|
||||
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(model_config)
|
||||
processor.info.get_supported_mm_limits = lambda: {"image": supported_limit}
|
||||
monkeypatch.setattr(
|
||||
processor.info, "get_supported_mm_limits", lambda: {"image": supported_limit}
|
||||
)
|
||||
|
||||
# This is what budget.py uses to derive mm_counts
|
||||
allowed = processor.info.allowed_mm_limits
|
||||
@@ -936,7 +939,7 @@ def test_hf_processor_init_kwargs(
|
||||
)
|
||||
|
||||
processor = ctx.get_hf_processor(
|
||||
DummyProcessor, # type: ignore[arg-type]
|
||||
DummyProcessor,
|
||||
**inference_kwargs,
|
||||
)
|
||||
assert processor.a == expected_kwargs["a"]
|
||||
@@ -967,7 +970,7 @@ def test_hf_processor_call_kwargs(
|
||||
tokenizer=None,
|
||||
)
|
||||
|
||||
processor = ctx.get_hf_processor(DummyProcessor) # type: ignore[arg-type]
|
||||
processor = ctx.get_hf_processor(DummyProcessor)
|
||||
|
||||
result = ctx.call_hf_processor(processor, {}, inference_kwargs)
|
||||
assert result == expected_kwargs
|
||||
@@ -1278,7 +1281,7 @@ def test_get_supported_mm_processor_kwargs_uses_supported_modalities(
|
||||
processor: SimpleNamespace,
|
||||
expected: dict[str, set[str]],
|
||||
) -> None:
|
||||
info = BaseProcessingInfo(SimpleNamespace())
|
||||
info = BaseProcessingInfo(SimpleNamespace()) # type: ignore[arg-type]
|
||||
info.__dict__["supported_mm_limits"] = supported_mm_limits
|
||||
monkeypatch.setattr(info, "get_hf_processor", lambda **_: processor)
|
||||
|
||||
@@ -1288,7 +1291,7 @@ def test_get_supported_mm_processor_kwargs_uses_supported_modalities(
|
||||
def test_supported_mm_processor_kwargs_is_cached(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
info = BaseProcessingInfo(SimpleNamespace())
|
||||
info = BaseProcessingInfo(SimpleNamespace()) # type: ignore[arg-type]
|
||||
expected = {"images_kwargs": {"size"}}
|
||||
calls = 0
|
||||
|
||||
@@ -1541,7 +1544,10 @@ def test_processor_inputs_hashes_scope_kwargs_by_modality():
|
||||
"video": [np.zeros((2, 8, 8, 3), dtype=np.uint8)],
|
||||
}
|
||||
)
|
||||
mm_uuid_items = {"image": ["image-uuid"], "video": ["video-uuid"]}
|
||||
mm_uuid_items: dict[str, Sequence[str | None]] = {
|
||||
"image": ["image-uuid"],
|
||||
"video": ["video-uuid"],
|
||||
}
|
||||
|
||||
def get_hashes(video_frames: int, image_size: int, video_size: int):
|
||||
return ProcessorInputs(
|
||||
@@ -1621,7 +1627,7 @@ def test_processor_inputs_hashes_distinguish_kwargs_shapes(left, right):
|
||||
[(None, 491520, 491520), (True, 491520, 8192), (True, 128, 128), (False, 128, 128)],
|
||||
)
|
||||
def test_dummy_inputs_scheduler_budget(
|
||||
chunked_prefill, max_model_len, expected_seq_len
|
||||
chunked_prefill, max_model_len, expected_seq_len, monkeypatch
|
||||
):
|
||||
ctx = build_model_context(
|
||||
"llava-hf/llava-v1.6-mistral-7b-hf",
|
||||
@@ -1634,11 +1640,9 @@ def test_dummy_inputs_scheduler_budget(
|
||||
ctx.model_config,
|
||||
tokenizer=ctx.tokenizer,
|
||||
)
|
||||
processor.apply = lambda *args, **kwargs: {"prompt_token_ids": [7]}
|
||||
|
||||
kwargs = {}
|
||||
scheduler_config = None
|
||||
if chunked_prefill is not None:
|
||||
kwargs["scheduler_config"] = SchedulerConfig(
|
||||
scheduler_config = SchedulerConfig(
|
||||
max_model_len=max_model_len,
|
||||
is_encoder_decoder=False,
|
||||
max_num_batched_tokens=8192,
|
||||
@@ -1646,5 +1650,10 @@ def test_dummy_inputs_scheduler_budget(
|
||||
enable_chunked_prefill=chunked_prefill,
|
||||
)
|
||||
|
||||
result = processor.get_dummy_mm_inputs({"image": 1}, **kwargs)
|
||||
monkeypatch.setattr(
|
||||
processor, "apply", lambda *args, **kwargs: {"prompt_token_ids": [7]}
|
||||
)
|
||||
result = processor.get_dummy_mm_inputs(
|
||||
{"image": 1}, scheduler_config=scheduler_config
|
||||
)
|
||||
assert len(result["prompt_token_ids"]) == expected_seq_len
|
||||
|
||||
@@ -24,4 +24,4 @@ def test_create_processor_error_uses_served_model_name():
|
||||
ValueError,
|
||||
match="friendly-model-name is not a multimodal model",
|
||||
):
|
||||
MULTIMODAL_REGISTRY.create_processor(model_config)
|
||||
MULTIMODAL_REGISTRY.create_processor(model_config) # type: ignore[arg-type]
|
||||
|
||||
@@ -7,6 +7,7 @@ import sys
|
||||
import threading
|
||||
from contextlib import ExitStack, contextmanager
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import numpy.typing as npt
|
||||
@@ -83,24 +84,28 @@ def _fresh_decoder_pool():
|
||||
@VIDEO_LOADER_REGISTRY.register("test_video_loader_1")
|
||||
class TestVideoLoader1(VideoLoader):
|
||||
@classmethod
|
||||
def load_bytes(cls, data: bytes, num_frames: int = -1) -> npt.NDArray:
|
||||
return FAKE_OUTPUT_1
|
||||
def load_bytes(
|
||||
cls, data: bytes, num_frames: int = -1, **kwargs
|
||||
) -> tuple[npt.NDArray, dict[str, Any]]:
|
||||
return FAKE_OUTPUT_1, {}
|
||||
|
||||
|
||||
@VIDEO_LOADER_REGISTRY.register("test_video_loader_2")
|
||||
class TestVideoLoader2(VideoLoader):
|
||||
@classmethod
|
||||
def load_bytes(cls, data: bytes, num_frames: int = -1) -> npt.NDArray:
|
||||
return FAKE_OUTPUT_2
|
||||
def load_bytes(
|
||||
cls, data: bytes, num_frames: int = -1, **kwargs
|
||||
) -> tuple[npt.NDArray, dict[str, Any]]:
|
||||
return FAKE_OUTPUT_2, {}
|
||||
|
||||
|
||||
def test_video_loader_registry():
|
||||
custom_loader_1 = VIDEO_LOADER_REGISTRY.load("test_video_loader_1")
|
||||
output_1 = custom_loader_1.load_bytes(b"test")
|
||||
output_1, _ = custom_loader_1.load_bytes(b"test")
|
||||
np.testing.assert_array_equal(output_1, FAKE_OUTPUT_1)
|
||||
|
||||
custom_loader_2 = VIDEO_LOADER_REGISTRY.load("test_video_loader_2")
|
||||
output_2 = custom_loader_2.load_bytes(b"test")
|
||||
output_2, _ = custom_loader_2.load_bytes(b"test")
|
||||
np.testing.assert_array_equal(output_2, FAKE_OUTPUT_2)
|
||||
|
||||
|
||||
@@ -488,7 +493,7 @@ def test_pynvvideocodec_failed_rebuild_invalidates_decoder_slot():
|
||||
|
||||
old_decoder = FakeDecoder()
|
||||
slot = PyNvVideoCodecDecoderSlot(FakeStream())
|
||||
slot.decoder = old_decoder
|
||||
slot.decoder = old_decoder # type: ignore[assignment] # Fake retained GPU decoder.
|
||||
slot.source_path = "valid.mp4"
|
||||
|
||||
class FakeNvc:
|
||||
@@ -596,7 +601,9 @@ def test_pynvvideocodec_h200_recovers_after_unsupported_8k():
|
||||
_pynv_decoder_pool.max_slots = old_max
|
||||
|
||||
|
||||
def test_pynvvideocodec_cross_subclass_shares_single_pool():
|
||||
def test_pynvvideocodec_cross_subclass_shares_single_pool(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""Regression test for GHSA-j682-9xp5-rrf3.
|
||||
|
||||
Multiple subclasses of PyNvVideoCodecVideoBackendMixin must share the
|
||||
@@ -623,11 +630,12 @@ def test_pynvvideocodec_cross_subclass_shares_single_pool():
|
||||
with _fresh_decoder_pool() as pool:
|
||||
pool.max_slots = 2
|
||||
|
||||
orig_create = PyNvVideoCodecVideoBackendMixin._create_decoder_slot
|
||||
PyNvVideoCodecVideoBackendMixin._create_decoder_slot = classmethod(
|
||||
fake_create_slot
|
||||
)
|
||||
try:
|
||||
with monkeypatch.context() as m:
|
||||
m.setattr(
|
||||
PyNvVideoCodecVideoBackendMixin,
|
||||
"_create_decoder_slot",
|
||||
classmethod(fake_create_slot),
|
||||
)
|
||||
with ExitStack() as stack:
|
||||
stack.enter_context(MixinSubclassA._borrow_decoder_slot())
|
||||
stack.enter_context(MixinSubclassB._borrow_decoder_slot())
|
||||
@@ -652,8 +660,6 @@ def test_pynvvideocodec_cross_subclass_shares_single_pool():
|
||||
|
||||
assert create_count == 2
|
||||
assert len(pool.slots) == 2
|
||||
finally:
|
||||
PyNvVideoCodecVideoBackendMixin._create_decoder_slot = orig_create
|
||||
|
||||
|
||||
@pytest.mark.parametrize("hw_decoders", [0, -1, 1.5, True, "2"])
|
||||
@@ -662,7 +668,7 @@ def test_pynvvideocodec_rejects_invalid_hw_decoders(hw_decoders: object):
|
||||
VideoBackend.load_bytes(
|
||||
b"fake video",
|
||||
backend=PYNVVIDEOCODEC_VIDEO_BACKEND,
|
||||
hw_decoders=hw_decoders, # type: ignore[arg-type]
|
||||
hw_decoders=hw_decoders,
|
||||
)
|
||||
|
||||
|
||||
@@ -799,6 +805,7 @@ def test_video_processor_from_model_repo(
|
||||
)
|
||||
|
||||
backend = get_video_loader_backend_for_processor(video_processor)
|
||||
assert backend is not None
|
||||
loader = VIDEO_LOADER_REGISTRY.load(backend)
|
||||
assert isinstance(loader, expected_loader_cls), (
|
||||
f"{model_repo!r}: backend={backend!r} loaded "
|
||||
@@ -1693,7 +1700,9 @@ def test_glm5next_backend_codec_parity(tmp_path, backend):
|
||||
path = _write_gray_video(tmp_path, total_frames, fps)
|
||||
# Dense default sampling (gap 5) and a sparse max_frames cap (gap 20).
|
||||
for max_frames in (None, 6):
|
||||
kwargs = {} if max_frames is None else {"max_frames": max_frames}
|
||||
kwargs: dict[str, Any] = (
|
||||
{} if max_frames is None else {"max_frames": max_frames}
|
||||
)
|
||||
expected = glm_sample_frame_indices(
|
||||
total_frames, float(fps), 12.0, max_frame_count=max_frames
|
||||
)
|
||||
|
||||
@@ -123,8 +123,8 @@ def test_auto_gptq_moe_creates_zero_initialized_expert_biases():
|
||||
method = object.__new__(AutoGPTQMoEMethod)
|
||||
method.quant_config = AutoGPTQConfig(4, 128, False, True, False, {}, {})
|
||||
method.input_dtype = None
|
||||
method.experts_cls = None
|
||||
method.moe = SimpleNamespace(w13_num_shards=2)
|
||||
method.experts_cls = None # type: ignore[assignment] # Weight creation does not select experts.
|
||||
method.moe = SimpleNamespace(w13_num_shards=2) # type: ignore[assignment]
|
||||
layer = torch.nn.Module()
|
||||
|
||||
method.create_weights(
|
||||
@@ -170,7 +170,7 @@ def test_routed_experts_loads_per_expert_biases():
|
||||
("w1", torch.tensor([1.0, 2.0, 3.0, 4.0])),
|
||||
("w3", torch.tensor([5.0, 6.0, 7.0, 8.0])),
|
||||
):
|
||||
assert RoutedExperts.weight_loader(
|
||||
assert RoutedExperts.weight_loader( # type: ignore[call-overload]
|
||||
loader,
|
||||
w13_bias,
|
||||
loaded,
|
||||
@@ -180,7 +180,7 @@ def test_routed_experts_loads_per_expert_biases():
|
||||
return_success=True,
|
||||
)
|
||||
|
||||
assert RoutedExperts.weight_loader(
|
||||
assert RoutedExperts.weight_loader( # type: ignore[call-overload]
|
||||
loader,
|
||||
w2_bias,
|
||||
torch.tensor([9.0, 10.0, 11.0, 12.0]),
|
||||
|
||||
@@ -193,7 +193,7 @@ class DummyFusedMoE:
|
||||
|
||||
|
||||
def make_config(**overrides) -> INCConfig:
|
||||
kwargs = {
|
||||
kwargs: dict[str, Any] = {
|
||||
"weight_bits": 4,
|
||||
"group_size": 128,
|
||||
"sym": True,
|
||||
@@ -208,7 +208,7 @@ def make_config(**overrides) -> INCConfig:
|
||||
|
||||
|
||||
def make_layer_config(**overrides) -> INCLayerConfig:
|
||||
kwargs = {
|
||||
kwargs: dict[str, Any] = {
|
||||
"bits": 4,
|
||||
"group_size": 128,
|
||||
"sym": True,
|
||||
@@ -423,7 +423,7 @@ def test_inc_get_quant_method_unquantized_routed_experts_with_model_prefix(
|
||||
|
||||
config = make_config(extra_config={"model.moe_layer": {"bits": 16}})
|
||||
layer = object.__new__(RoutedExperts)
|
||||
layer.moe_config = None
|
||||
layer.moe_config = None # type: ignore[assignment] # No MoE config used by this stub.
|
||||
|
||||
method = config.get_quant_method(layer, "moe_layer")
|
||||
|
||||
@@ -730,7 +730,7 @@ def test_wna16_cuda_low_bit_moe_routes_to_humming(monkeypatch, bits) -> None:
|
||||
)
|
||||
|
||||
layer = object.__new__(RoutedExperts)
|
||||
layer.moe_config = DummyMoeConfig()
|
||||
layer.moe_config = DummyMoeConfig() # type: ignore[assignment]
|
||||
layer_config = make_layer_config(bits=bits)
|
||||
method = INCWna16Scheme().get_moe_method(
|
||||
make_config(), layer, "model.layers.0.mlp", layer_config
|
||||
@@ -861,7 +861,7 @@ def test_qwen3_30b_a3b_w4a16_autoround_routes_to_gptq_moe(
|
||||
|
||||
config = make_qwen3_autoround_config("qwen3_30b_a3b_w4a16")
|
||||
layer = object.__new__(RoutedExperts)
|
||||
layer.moe_config = DummyMoeConfig()
|
||||
layer.moe_config = DummyMoeConfig() # type: ignore[assignment]
|
||||
|
||||
method = config.get_quant_method(layer, "model.layers.0.mlp")
|
||||
|
||||
@@ -890,7 +890,7 @@ def test_qwen3_30b_a3b_mxfp4_autoround_routes_to_mxfp4_moe(
|
||||
|
||||
config = make_qwen3_autoround_config("qwen3_30b_a3b_mxfp4")
|
||||
layer = object.__new__(RoutedExperts)
|
||||
layer.moe_config = DummyMoeConfig()
|
||||
layer.moe_config = DummyMoeConfig() # type: ignore[assignment]
|
||||
|
||||
ignored_method = config.get_quant_method(
|
||||
object.__new__(LinearBase), "model.layers.0.self_attn.q_proj"
|
||||
@@ -1186,7 +1186,7 @@ def test_wna16_xpu_moe_routes_to_gptq_moe(monkeypatch) -> None:
|
||||
)
|
||||
|
||||
layer = object.__new__(RoutedExperts)
|
||||
layer.moe_config = DummyMoeConfig()
|
||||
layer.moe_config = DummyMoeConfig() # type: ignore[assignment]
|
||||
method = INCWna16Scheme().get_moe_method(
|
||||
make_config(),
|
||||
layer,
|
||||
@@ -1669,7 +1669,7 @@ def test_inc_get_quant_method_unquantized_moe_returns_unquantized(
|
||||
when extra_config has bits >= 16."""
|
||||
config = make_config(extra_config={"layer": {"bits": 16}})
|
||||
layer = object.__new__(RoutedExperts)
|
||||
layer.moe_config = None # UnquantizedFusedMoEMethod accepts moe_config
|
||||
layer.moe_config = None # type: ignore[assignment] # No MoE config used by this stub.
|
||||
|
||||
class DummyUnquantizedFusedMoEMethod:
|
||||
def __init__(self, moe_config) -> None:
|
||||
@@ -2094,7 +2094,7 @@ def _with_w4a8_kernel(monkeypatch) -> None:
|
||||
|
||||
def _dispatch(layer_config=None):
|
||||
return INCWna16Scheme().get_linear_method(
|
||||
object(), object(), "layer", layer_config or make_layer_config()
|
||||
make_config(), object(), "layer", layer_config or make_layer_config()
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -99,6 +99,7 @@ def test_machete_rejects_invalid_config(
|
||||
)
|
||||
can_impl, reason = MacheteLinearKernel.can_implement(config)
|
||||
assert not can_impl
|
||||
assert reason is not None
|
||||
assert expected_reason in reason
|
||||
|
||||
|
||||
|
||||
@@ -25,6 +25,7 @@ from vllm.model_executor.layers.attention.attention import (
|
||||
set_default_quant_scales,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
|
||||
from vllm.model_executor.layers.quantization.fp8 import (
|
||||
Fp8Config,
|
||||
Fp8LinearMethod,
|
||||
@@ -79,17 +80,22 @@ def test_deepseek_v41_mxfp8_scale_loading(
|
||||
for module in (model_module, linear_module):
|
||||
monkeypatch.setattr(module, "get_tensor_model_parallel_world_size", lambda: 2)
|
||||
monkeypatch.setattr(module, "get_tensor_model_parallel_rank", lambda: tp_rank)
|
||||
tp_args = dict(quant_config=quant_config, bias=False)
|
||||
block = torch.nn.Module()
|
||||
block.attn = torch.nn.Module()
|
||||
block.attn.wq_b = ColumnParallelLinear(128, 128, **tp_args)
|
||||
block.attn.fused_wqa_wkv = MergedColumnParallelLinear(128, [128, 64], **tp_args)
|
||||
block.attn.wq_b = ColumnParallelLinear(
|
||||
128, 128, quant_config=quant_config, bias=False
|
||||
)
|
||||
block.attn.fused_wqa_wkv = MergedColumnParallelLinear(
|
||||
128, [128, 64], quant_config=quant_config, bias=False
|
||||
)
|
||||
block.ffn = torch.nn.Module()
|
||||
block.ffn.shared_experts = torch.nn.Module()
|
||||
block.ffn.shared_experts.gate_up_proj = MergedColumnParallelLinear(
|
||||
128, [128, 128], **tp_args
|
||||
128, [128, 128], quant_config=quant_config, bias=False
|
||||
)
|
||||
block.ffn.shared_experts.down_proj = RowParallelLinear(
|
||||
128, 128, quant_config=quant_config, bias=False
|
||||
)
|
||||
block.ffn.shared_experts.down_proj = RowParallelLinear(128, 128, **tp_args)
|
||||
block.ffn.experts = torch.nn.Module()
|
||||
block.ffn.experts.register_parameter(
|
||||
"weight_scale", torch.nn.Parameter(torch.empty(3, 4, dtype=torch.uint8), False)
|
||||
@@ -111,9 +117,11 @@ def test_deepseek_v41_mxfp8_scale_loading(
|
||||
model.config = SimpleNamespace(num_attention_heads=2)
|
||||
model.quant_config = quant_config
|
||||
model.use_sequence_parallel = False
|
||||
model.get_expert_mapping = lambda: [
|
||||
("experts.weight_scale", "experts.0.w1.weight_scale", 0, "w1")
|
||||
]
|
||||
monkeypatch.setattr(
|
||||
model,
|
||||
"get_expert_mapping",
|
||||
lambda: [("experts.weight_scale", "experts.0.w1.weight_scale", 0, "w1")],
|
||||
)
|
||||
|
||||
checkpoints = [
|
||||
("attn.wq_b", "attn.wq_b", 128, 128, 0, None),
|
||||
@@ -211,7 +219,10 @@ def test_deepseek_v41_vl_mapper_routes_linear_scales(
|
||||
vllm_config = SimpleNamespace(
|
||||
quant_config=SimpleNamespace(weight_block_size=weight_block_size)
|
||||
)
|
||||
resolved = model_module._linear_scale_param_name(vllm_config, expert_dtype)
|
||||
resolved = model_module._linear_scale_param_name(
|
||||
vllm_config, # type: ignore[arg-type]
|
||||
expert_dtype,
|
||||
)
|
||||
assert resolved == scale_name
|
||||
|
||||
mapper = vl_module._make_deepseek_v4_vl_weights_mapper(expert_dtype, resolved)
|
||||
@@ -373,7 +384,7 @@ def test_deepgemm_mxfp8_preserves_weight_and_scale_values(
|
||||
@pytest.mark.parametrize("config_source", ["deepseek", "mxfp8"])
|
||||
@pytest.mark.parametrize("num_tokens", [1, 7, 128])
|
||||
def test_mxfp8_bmm_loads_and_projects_grouped_weights(
|
||||
dist_init, default_vllm_config, prequantized, config_source, num_tokens
|
||||
dist_init, default_vllm_config, monkeypatch, prequantized, config_source, num_tokens
|
||||
):
|
||||
"""BMM metadata set after construction selects grouped weight processing."""
|
||||
from vllm.model_executor.kernels.linear.mxfp8.deep_gemm import (
|
||||
@@ -398,10 +409,11 @@ def test_mxfp8_bmm_loads_and_projects_grouped_weights(
|
||||
pytest.skip("DeepGEMM MXFP8 BMM requires Blackwell")
|
||||
|
||||
default_vllm_config.model_config = SimpleNamespace(dtype=torch.bfloat16)
|
||||
quant_config = DeepseekV4FP8Config(
|
||||
deepseek_config = DeepseekV4FP8Config(
|
||||
is_checkpoint_fp8_serialized=True, weight_block_size=[32, 32]
|
||||
)
|
||||
quant_config._resolved_expert_dtype = "fp4"
|
||||
deepseek_config._resolved_expert_dtype = "fp4"
|
||||
quant_config: QuantizationConfig = deepseek_config
|
||||
if config_source == "mxfp8":
|
||||
quant_config = get_quantization_config("mxfp8").from_config(
|
||||
{"quant_method": "mxfp8"}
|
||||
@@ -432,7 +444,7 @@ def test_mxfp8_bmm_loads_and_projects_grouped_weights(
|
||||
model.config = SimpleNamespace(num_attention_heads=2)
|
||||
model.quant_config = quant_config
|
||||
model.use_sequence_parallel = False
|
||||
model.get_expert_mapping = lambda: []
|
||||
monkeypatch.setattr(model, "get_expert_mapping", lambda: [])
|
||||
model.load_weights(
|
||||
[
|
||||
("layers.0.attn.wo_a.weight", weight),
|
||||
@@ -443,6 +455,7 @@ def test_mxfp8_bmm_loads_and_projects_grouped_weights(
|
||||
weight.float().reshape(8, 32, 16, 32)
|
||||
* torch.exp2(scales.float() - 127)[:, None, :, None]
|
||||
).reshape(2, 128, 512)
|
||||
assert isinstance(linear.quant_method, ModelOptLinearMethod)
|
||||
linear.quant_method.process_weights_after_loading(linear)
|
||||
linear.quant_method.process_weights_after_loading(linear)
|
||||
assert isinstance(linear.quant_method.kernel, DeepGemmMxfp8BmmLinearKernel)
|
||||
@@ -785,7 +798,7 @@ def test_kv_cache_scale_sync_to_host_copies():
|
||||
set_default_quant_scales(layer, register_buffer=True)
|
||||
layer.kv_cache_dtype = "fp8"
|
||||
|
||||
method = BaseKVCacheMethod(quant_config=None)
|
||||
method = BaseKVCacheMethod(quant_config=None) # type: ignore[arg-type] # Scale synchronization does not use quant_config.
|
||||
method.create_weights(layer)
|
||||
# 0.3 stays != 1.0 even after the fp8_fnuz x2 rescale.
|
||||
checkpoint_scale = torch.tensor(0.3, dtype=torch.float32)
|
||||
|
||||
@@ -8,6 +8,7 @@ These tests run on real hardware — no mocks. Skipped on non-GFX950 platforms.
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config.kernel import MoEBackend
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEParallelConfig,
|
||||
@@ -53,7 +54,7 @@ def disable_rocm_aiter(monkeypatch: pytest.MonkeyPatch):
|
||||
set_rocm_aiter(monkeypatch, False)
|
||||
|
||||
|
||||
def _make_w4a4_moe_config(moe_backend: str = "auto") -> FusedMoEConfig:
|
||||
def _make_w4a4_moe_config(moe_backend: MoEBackend = "auto") -> FusedMoEConfig:
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
|
||||
return FusedMoEConfig(
|
||||
|
||||
@@ -10,6 +10,7 @@ CUDA (SM >= 7.5), ROCm, or XPU — not just gfx950.
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config.kernel import MoEBackend
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEParallelConfig,
|
||||
@@ -44,7 +45,7 @@ requires_int8_moe = pytest.mark.skipif(
|
||||
DEVICE = current_platform.device_type
|
||||
|
||||
|
||||
def _make_int8_moe_config(moe_backend: str = "auto") -> FusedMoEConfig:
|
||||
def _make_int8_moe_config(moe_backend: MoEBackend = "auto") -> FusedMoEConfig:
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
|
||||
return FusedMoEConfig(
|
||||
|
||||
@@ -15,6 +15,7 @@ from tests.quantization.utils import load_model_without_vllm_runner
|
||||
from vllm.model_executor.layers.linear import ColumnParallelLinear
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQLinearMethod
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501
|
||||
CompressedTensorsConfig,
|
||||
)
|
||||
@@ -80,6 +81,7 @@ def test_quantized_lm_head_matches_linear(
|
||||
dtype=torch.bfloat16, head_dtype=None
|
||||
)
|
||||
default_vllm_config.kernel_config.linear_backend = "humming"
|
||||
quant_config: QuantizationConfig
|
||||
if quant_format is None:
|
||||
quant_config = ModelOptNvFp4Config(
|
||||
quant_method=preset,
|
||||
|
||||
@@ -843,6 +843,7 @@ def test_modelopt_mixed_precision_builds_w4a16_sibling_config():
|
||||
}
|
||||
config = m.ModelOptMixedPrecisionConfig.from_config(hf_quant_config)
|
||||
|
||||
assert isinstance(config, m.ModelOptMixedPrecisionConfig)
|
||||
assert config.nvfp4_config.quant_method == "NVFP4"
|
||||
assert config.w4a16_nvfp4_config.quant_method == "W4A16_NVFP4"
|
||||
|
||||
|
||||
@@ -218,11 +218,13 @@ def test_gptq_nfirst_weights_pass_through_for_triton():
|
||||
|
||||
def test_moe_wna16_setup_forwards_selected_backend(monkeypatch):
|
||||
method = object.__new__(MoeWNA16Method)
|
||||
method.experts_cls = object
|
||||
method.experts_cls = object # type: ignore[assignment] # Opaque mocked-expert sentinel.
|
||||
method.wna16_backend = WNA16MoEBackend.HUMMING
|
||||
method.moe = object()
|
||||
method.moe = object() # type: ignore[assignment] # Opaque mocked-config sentinel.
|
||||
quant_config = object()
|
||||
method.get_fused_moe_quant_config = lambda layer: quant_config
|
||||
monkeypatch.setattr(
|
||||
method, "get_fused_moe_quant_config", lambda layer: quant_config
|
||||
)
|
||||
layer = SimpleNamespace(_expert_routing_tables=lambda: (None, None, None))
|
||||
captured = {}
|
||||
kernel = object()
|
||||
@@ -233,7 +235,7 @@ def test_moe_wna16_setup_forwards_selected_backend(monkeypatch):
|
||||
|
||||
monkeypatch.setattr(moe_wna16, "make_wna16_moe_kernel", fake_make_wna16_moe_kernel)
|
||||
|
||||
method._setup_kernel(layer)
|
||||
method._setup_kernel(layer) # type: ignore[arg-type] # Routing-table stub.
|
||||
|
||||
assert method.moe_kernel is kernel
|
||||
assert captured["backend"] == WNA16MoEBackend.HUMMING
|
||||
@@ -278,7 +280,7 @@ def test_moe_wna16_uses_humming_quant_config(monkeypatch):
|
||||
),
|
||||
)
|
||||
|
||||
assert method.get_fused_moe_quant_config(layer) is quant_config
|
||||
assert method.get_fused_moe_quant_config(layer) is quant_config # type: ignore[arg-type] # Identity sentinel.
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
|
||||
@@ -16,6 +16,7 @@ from tests.quantization.utils import (
|
||||
_test_online_quant_peak_mem_impl,
|
||||
is_quant_method_supported,
|
||||
load_model_without_vllm_runner,
|
||||
quant_config_args,
|
||||
)
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
@@ -180,7 +181,7 @@ def test_qianfan_online_fp8_keeps_vision_layers_unquantized(
|
||||
config = OnlineQuantizationConfig(args)
|
||||
other_config = OnlineQuantizationConfig(args)
|
||||
model = QianfanOCRForConditionalGeneration.__new__(
|
||||
QianfanOCRForConditionalGeneration
|
||||
QianfanOCRForConditionalGeneration # type: ignore[type-abstract]
|
||||
)
|
||||
model._patch_quant_config(
|
||||
SimpleNamespace(vision_config=SimpleNamespace(num_hidden_layers=2)),
|
||||
@@ -209,9 +210,9 @@ def test_online_nvfp4_reuses_kernel_when_weights_are_reprocessed(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
method = object.__new__(Nvfp4OnlineMoEMethod)
|
||||
method.moe = SimpleNamespace(is_act_and_mul=True)
|
||||
method.nvfp4_backend = object()
|
||||
method.experts_cls = object
|
||||
method.moe = SimpleNamespace(is_act_and_mul=True) # type: ignore[assignment]
|
||||
method.nvfp4_backend = object() # type: ignore[assignment] # Opaque mocked-backend sentinel.
|
||||
method.experts_cls = object # type: ignore[assignment] # Opaque mocked-expert sentinel.
|
||||
method.moe_quant_config = None
|
||||
method.moe_kernel = None
|
||||
|
||||
@@ -239,7 +240,7 @@ def test_online_nvfp4_reuses_kernel_when_weights_are_reprocessed(
|
||||
)
|
||||
make_kernel = Mock(return_value=kernel)
|
||||
get_quant_config = Mock(return_value=quant_config)
|
||||
method.get_fused_moe_quant_config = get_quant_config
|
||||
monkeypatch.setattr(method, "get_fused_moe_quant_config", get_quant_config)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.layers.quantization.online.nvfp4."
|
||||
@@ -355,7 +356,7 @@ def test_online_prequantized_compatibility(
|
||||
checkpoint_config = checkpoint_config_factory()
|
||||
|
||||
checkpoint_config.online_quantization_config = OnlineQuantizationConfig(
|
||||
QuantizationConfigArgs(linear="mxfp8")
|
||||
quant_config_args(linear="mxfp8")
|
||||
)
|
||||
config = checkpoint_config
|
||||
|
||||
@@ -408,7 +409,7 @@ def test_online_ignore_keeps_checkpoint_quantization_linear(
|
||||
quant_config = _fully_quantized_quark_config()
|
||||
prefix = "model.layers.0.self_attn.o_proj"
|
||||
quant_config.online_quantization_config = OnlineQuantizationConfig(
|
||||
QuantizationConfigArgs(linear="mxfp8", ignore=[prefix])
|
||||
quant_config_args(linear="mxfp8", ignore=[prefix])
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
quant_config.online_quantization_config,
|
||||
@@ -437,7 +438,7 @@ def test_online_quantization_rejects_prequantized_moe(
|
||||
prefix = "model.layers.0.mlp.experts"
|
||||
quant_config = _fully_quantized_quark_config()
|
||||
quant_config.online_quantization_config = OnlineQuantizationConfig(
|
||||
QuantizationConfigArgs(linear="mxfp4", moe="mxfp4")
|
||||
quant_config_args(linear="mxfp4", moe="mxfp4")
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="pre-quantized layer"):
|
||||
@@ -460,7 +461,7 @@ def test_activation_only_override_applies_to_checkpoint_method(
|
||||
model_config = ModelConfig(
|
||||
model=str(tmp_path),
|
||||
quantization="compressed-tensors",
|
||||
quantization_config=QuantizationConfigArgs(moe={"activation": "mxfp8"}),
|
||||
quantization_config=quant_config_args(moe={"activation": "mxfp8"}),
|
||||
hf_overrides={
|
||||
"quantization_config": {
|
||||
"quant_method": "compressed-tensors",
|
||||
@@ -514,7 +515,7 @@ def test_online_overlay_loads_checkpoint_config_file(tmp_path) -> None:
|
||||
model_config = ModelConfig(
|
||||
model=str(tmp_path),
|
||||
quantization="awq",
|
||||
quantization_config=QuantizationConfigArgs(linear="mxfp8"),
|
||||
quantization_config=quant_config_args(linear="mxfp8"),
|
||||
)
|
||||
|
||||
result = weight_utils.get_quant_config(model_config, LoadConfig())
|
||||
@@ -574,7 +575,7 @@ def test_online_shorthand_selects_checkpoint_or_online_config(
|
||||
|
||||
def test_log_online_quantization_for_composable_config(monkeypatch) -> None:
|
||||
"""Composable configs log their nested online quantization results."""
|
||||
online_config = OnlineQuantizationConfig(QuantizationConfigArgs(linear="mxfp8"))
|
||||
online_config = OnlineQuantizationConfig(quant_config_args(linear="mxfp8"))
|
||||
online_config.quantized_layers = {
|
||||
"model.layers.0.self_attn.o_proj": ("linear", "mxfp8", None),
|
||||
"model.layers.1.self_attn.o_proj": ("linear", "mxfp8", None),
|
||||
@@ -1164,7 +1165,7 @@ def test_online_quantization_records_global_config(
|
||||
default_vllm_config, dist_init
|
||||
) -> None:
|
||||
default_vllm_config.model_config = ModelConfig()
|
||||
config = OnlineQuantizationConfig(QuantizationConfigArgs(linear="fp8_per_block"))
|
||||
config = OnlineQuantizationConfig(quant_config_args(linear="fp8_per_block"))
|
||||
prefix = "model.layers.0.self_attn.o_proj"
|
||||
layer = ColumnParallelLinear(
|
||||
input_size=1,
|
||||
@@ -1218,7 +1219,7 @@ def test_online_quantization_targets_reject_unsupported_layer() -> None:
|
||||
|
||||
|
||||
def test_log_online_quantization(default_vllm_config, monkeypatch) -> None:
|
||||
config = OnlineQuantizationConfig(QuantizationConfigArgs(linear="fp8_per_tensor"))
|
||||
config = OnlineQuantizationConfig(quant_config_args(linear="fp8_per_tensor"))
|
||||
config.quantized_layers = {
|
||||
"model.layers.0.mlp.down_proj": ("linear", "fp8_per_tensor", None),
|
||||
"model.layers.1.mlp.down_proj": ("linear", "fp8_per_tensor", None),
|
||||
@@ -1431,7 +1432,7 @@ def test_online_int8_moe_w2_scale_matches_unsharded(monkeypatch) -> None:
|
||||
layer.w2_weight = torch.nn.Parameter(w2_in, requires_grad=False)
|
||||
layer.num_experts = layer.local_num_experts = w13.shape[0]
|
||||
method = SimpleNamespace(moe=SimpleNamespace(tp_size=moe_tp_size))
|
||||
Int8OnlineMoEMethod._quantize_weights(method, layer)
|
||||
Int8OnlineMoEMethod._quantize_weights(method, layer) # type: ignore[arg-type]
|
||||
return layer.w2_weight, layer.w2_scale
|
||||
|
||||
full_weight, full_scale = quantize(w2, 1)
|
||||
|
||||
@@ -10,6 +10,7 @@ against the pure-torch reference in `reference_mxfp4.py`.
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config.kernel import MoEBackend
|
||||
from vllm.config.model import ModelConfig
|
||||
from vllm.model_executor.kernels.linear import _POSSIBLE_MXFP4_KERNELS
|
||||
from vllm.model_executor.kernels.linear.mxfp4.aiter import (
|
||||
@@ -282,7 +283,7 @@ def test_online_mxfp4_tp_weight_quant_matches_unsharded(tp_size: int):
|
||||
],
|
||||
)
|
||||
def test_online_mxfp4_moe_matches_quark(
|
||||
moe_backend: str,
|
||||
moe_backend: MoEBackend,
|
||||
unpadded_hidden_size: int,
|
||||
unpadded_intermediate_size: int,
|
||||
default_vllm_config,
|
||||
|
||||
@@ -6,6 +6,7 @@ from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.quantization.utils import quant_config_args, quant_spec
|
||||
from vllm.config.quantization import (
|
||||
QUANT_KEY_NAMES,
|
||||
QuantizationConfigArgs,
|
||||
@@ -30,7 +31,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
|
||||
|
||||
def test_quant_spec_resolves_string_to_quant_key():
|
||||
spec = QuantSpec(weight="mxfp8", activation="fp8_per_token")
|
||||
spec = quant_spec(weight="mxfp8", activation="fp8_per_token")
|
||||
assert spec.weight == kMxfp8Dynamic
|
||||
assert spec.activation == kFp8DynamicTokenSym
|
||||
|
||||
@@ -42,12 +43,12 @@ def test_quant_spec_accepts_quant_key_directly():
|
||||
|
||||
|
||||
def test_quant_spec_string_representation_uses_quantization_name():
|
||||
assert str(QuantSpec(weight="mxfp4")) == "mxfp4"
|
||||
assert str(quant_spec(weight="mxfp4")) == "mxfp4"
|
||||
|
||||
|
||||
def test_quant_spec_rejects_unknown_name():
|
||||
with pytest.raises(ValueError, match="unknown quantization name"):
|
||||
QuantSpec(weight="not_a_real_format")
|
||||
quant_spec(weight="not_a_real_format")
|
||||
|
||||
|
||||
# ---- QuantizationConfigArgs string shorthand on linear/moe ----------------
|
||||
@@ -55,7 +56,7 @@ def test_quant_spec_rejects_unknown_name():
|
||||
|
||||
def test_args_linear_string_resolves_via_quant_key_names():
|
||||
# A bare QUANT_KEY_NAMES entry desugars to QuantSpec(weight=<key>).
|
||||
args = QuantizationConfigArgs(linear="fp8_per_block_static")
|
||||
args = quant_config_args(linear="fp8_per_block_static")
|
||||
assert args.linear == QuantSpec(weight=kFp8Static128BlockSym)
|
||||
assert args.moe is None
|
||||
|
||||
@@ -65,7 +66,7 @@ def test_args_moe_string_resolves_via_online_shorthand():
|
||||
# (so `linear: "fp8_per_block"` and `moe: "fp8_per_block"` produce the
|
||||
# same per-layer-kind spec the `--quantization fp8_per_block` shorthand
|
||||
# would).
|
||||
args = QuantizationConfigArgs(moe="fp8_per_block")
|
||||
args = quant_config_args(moe="fp8_per_block")
|
||||
assert args.moe == QuantSpec(weight=kFp8Static128BlockSym)
|
||||
|
||||
|
||||
@@ -73,11 +74,11 @@ def test_args_string_shorthand_missing_slot_raises():
|
||||
# int8_per_channel_weight_only sets only `moe`; using it on `linear`
|
||||
# has no defined spec and should raise rather than silently no-op.
|
||||
with pytest.raises(ValueError, match="does not define a linear spec"):
|
||||
QuantizationConfigArgs(linear="int8_per_channel_weight_only")
|
||||
quant_config_args(linear="int8_per_channel_weight_only")
|
||||
|
||||
|
||||
def test_args_accepts_dict_form():
|
||||
args = QuantizationConfigArgs(moe={"activation": "mxfp8"})
|
||||
args = quant_config_args(moe={"activation": "mxfp8"})
|
||||
assert args.moe == QuantSpec(weight=None, activation=kMxfp8Dynamic)
|
||||
|
||||
|
||||
@@ -91,6 +92,7 @@ def test_targets_reject_non_string_keys():
|
||||
|
||||
def test_resolve_shorthand_only_populates_both_slots():
|
||||
args = resolve_quantization_config("fp8_per_block", None)
|
||||
assert args is not None
|
||||
assert args.linear == QuantSpec(weight=kFp8Static128BlockSym)
|
||||
assert args.moe == QuantSpec(weight=kFp8Static128BlockSym)
|
||||
|
||||
@@ -105,6 +107,7 @@ def test_resolve_int8_shorthand_leaves_linear_unset():
|
||||
# int8_per_channel_weight_only is MoE-only; linear stays None so that
|
||||
# OnlineQuantizationConfig leaves Linear layers in full precision.
|
||||
args = resolve_quantization_config("int8_per_channel_weight_only", None)
|
||||
assert args is not None
|
||||
assert args.linear is None
|
||||
assert args.moe == QuantSpec(weight=kInt8StaticChannelSym)
|
||||
|
||||
@@ -113,6 +116,7 @@ def test_resolve_quantization_config_only():
|
||||
# When only `quantization_config` is given (e.g. for an already-quantized
|
||||
# checkpoint that needs an activation override), it's returned as-is.
|
||||
args = resolve_quantization_config(None, {"moe": {"activation": "mxfp8"}})
|
||||
assert args is not None
|
||||
assert args.linear is None
|
||||
assert args.moe == QuantSpec(weight=None, activation=kMxfp8Dynamic)
|
||||
|
||||
@@ -124,13 +128,14 @@ def test_resolve_merges_explicit_over_shorthand():
|
||||
"fp8_per_tensor",
|
||||
{"linear": "fp8_per_block"},
|
||||
)
|
||||
assert args is not None
|
||||
assert args.linear == QuantSpec(weight=kFp8Static128BlockSym)
|
||||
assert args.moe == QuantSpec(weight=kFp8StaticTensorSym)
|
||||
|
||||
|
||||
def test_resolve_quantization_config_with_checkpoint_quantization():
|
||||
args = resolve_quantization_config("gptq", {"linear": "fp8_per_block"})
|
||||
assert args == QuantizationConfigArgs(linear="fp8_per_block")
|
||||
assert args == quant_config_args(linear="fp8_per_block")
|
||||
|
||||
|
||||
# ---- QUANT_KEY_NAMES coverage --------------------------------------------
|
||||
@@ -140,15 +145,15 @@ def test_quant_key_names_round_trip():
|
||||
# Every advertised name should round-trip through QuantSpec without error
|
||||
# and produce the same QuantKey it maps to.
|
||||
for name, expected in QUANT_KEY_NAMES.items():
|
||||
assert QuantSpec(weight=name).weight == expected, name
|
||||
assert QuantSpec(activation=name).activation == expected, name
|
||||
assert quant_spec(weight=name).weight == expected, name
|
||||
assert quant_spec(activation=name).activation == expected, name
|
||||
|
||||
|
||||
def test_static_block_weight_paired_with_dynamic_block_activation():
|
||||
# The block-FP8 shorthand pair: 128x128 static weights + 1x128 dynamic
|
||||
# activations. Pinning this so renames in QUANT_KEY_NAMES don't quietly
|
||||
# rewire the kernel dispatch.
|
||||
spec = QuantSpec(weight="fp8_per_block_static", activation="fp8_per_block_dynamic")
|
||||
spec = quant_spec(weight="fp8_per_block_static", activation="fp8_per_block_dynamic")
|
||||
assert spec.weight == kFp8Static128BlockSym
|
||||
assert spec.activation == kFp8Dynamic128Sym
|
||||
|
||||
|
||||
@@ -22,7 +22,7 @@ from packaging import version
|
||||
from tests.quantization.utils import load_model_without_vllm_runner
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.config.cache import CacheConfig
|
||||
from vllm.config.cache import CacheConfig, CacheDType
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.model_executor import parameter
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.aiter import (
|
||||
@@ -1081,6 +1081,7 @@ def test_quant_method_dispatch_mxfp8_moe_raises(default_vllm_config):
|
||||
)
|
||||
wk, ak, mcls = config.get_quant_method_target("experts", RoutedExperts)
|
||||
assert mcls is QuarkOCP_MX_MoEMethod
|
||||
assert wk is not None
|
||||
# The OCP MX MoE constructor should fail loudly for MXFP8.
|
||||
fake_moe_config = MagicMock()
|
||||
with pytest.raises(ValueError, match="MXFP8 experts are not supported"):
|
||||
@@ -1321,7 +1322,6 @@ def test_quark_fp8_ptpc_exposes_kernel_input_quant_key(monkeypatch):
|
||||
scheme = QuarkW8A8Fp8.__new__(QuarkW8A8Fp8)
|
||||
scheme.weight_qscheme = "per_channel"
|
||||
scheme.is_static_input_scheme = False
|
||||
scheme.input_qscheme = "per_channel"
|
||||
scheme.activation_quant_key = kFp8DynamicTokenSym
|
||||
scheme.weight_quant_key = kFp8StaticChannelSym
|
||||
scheme.out_dtype = dtype
|
||||
@@ -1343,7 +1343,7 @@ def test_quark_fp8_ptpc_exposes_kernel_input_quant_key(monkeypatch):
|
||||
|
||||
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
|
||||
def test_quark_fp8_w_per_tensor_a_per_tensor(
|
||||
kv_cache_dtype: str, monkeypatch, dist_init, workspace_init
|
||||
kv_cache_dtype: CacheDType, monkeypatch, dist_init, workspace_init
|
||||
):
|
||||
model_path = "amd/Llama-3.1-8B-Instruct-FP8-KV-Quark-test"
|
||||
checkpoint_scales = {}
|
||||
@@ -2314,7 +2314,7 @@ class TestQuarkInt4Format:
|
||||
None,
|
||||
quant_config.quant_config["global_quant_config"]["weight"],
|
||||
quant_config.pack_method,
|
||||
moe_config,
|
||||
moe_config, # type: ignore[arg-type]
|
||||
)
|
||||
|
||||
layer = _FakeLayer(moe_config)
|
||||
@@ -2325,7 +2325,10 @@ class TestQuarkInt4Format:
|
||||
torch.zeros(1, 16, 8 // tp_size, dtype=torch.uint8),
|
||||
requires_grad=False,
|
||||
)
|
||||
loader = method.get_weight_loader(layer, weight_loader=None)
|
||||
loader = method.get_weight_loader(
|
||||
layer, # type: ignore[arg-type]
|
||||
weight_loader=None,
|
||||
)
|
||||
loader(
|
||||
param,
|
||||
raw_zp.clone(),
|
||||
|
||||
@@ -75,7 +75,7 @@ class CustomQuantConfig(QuantizationConfig):
|
||||
|
||||
def get_name(self) -> QuantizationMethods:
|
||||
"""Name of the quantization method."""
|
||||
return "custom_quant"
|
||||
return "custom_quant" # type: ignore[return-value] # Runtime registration extends built-in names.
|
||||
|
||||
def get_supported_act_dtypes(self) -> list[torch.dtype]:
|
||||
"""List of supported activation dtypes."""
|
||||
|
||||
@@ -12,6 +12,7 @@ from vllm.config import set_current_vllm_config
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.model_loader import get_model_loader
|
||||
from vllm.model_executor.model_loader.default_loader import DefaultModelLoader
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if current_platform.is_rocm():
|
||||
@@ -126,6 +127,7 @@ def test_online_quant_config_dict_json(vllm_runner, enable_pickle):
|
||||
|
||||
def load_weights(model):
|
||||
model_loader = get_model_loader(load_config)
|
||||
assert isinstance(model_loader, DefaultModelLoader)
|
||||
weights_iterator = model_loader.get_all_weights(model_config, model)
|
||||
model.load_weights(weights_iterator)
|
||||
|
||||
|
||||
@@ -34,7 +34,7 @@ def test_shared_nvfp4_input_scales_have_writable_storage(monkeypatch):
|
||||
|
||||
outputs = prepare_nvfp4_moe_layer_for_fi_or_cutlass(
|
||||
backend=NvFp4MoeBackend.FLASHINFER_CUTLASS,
|
||||
layer=layer,
|
||||
layer=layer, # type: ignore[arg-type]
|
||||
w13=w13,
|
||||
w13_scale=w13_scale,
|
||||
w13_scale_2=weight_scale,
|
||||
|
||||
@@ -300,7 +300,7 @@ class TestTurboQuantKVCacheSpec:
|
||||
|
||||
# The layer builds an unpacked spec; the worker's spec-collection
|
||||
# loop applies TQ slot packing via the backend's customize_spec hook.
|
||||
spec = Attention.get_kv_cache_spec(layer, vllm_config)
|
||||
spec = Attention.get_kv_cache_spec(layer, vllm_config) # type: ignore[arg-type]
|
||||
assert isinstance(spec, FullAttentionSpec)
|
||||
assert spec.kv_quant_mode.is_turboquant
|
||||
assert spec.state_content_bytes is None
|
||||
|
||||
@@ -6,18 +6,31 @@ from typing import Any
|
||||
|
||||
import regex as re
|
||||
import torch
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from vllm.config import (
|
||||
CompilationConfig,
|
||||
CompilationMode,
|
||||
ModelConfig,
|
||||
VllmConfig,
|
||||
set_current_vllm_config,
|
||||
)
|
||||
from vllm.config.quantization import QuantizationConfigArgs, QuantSpec
|
||||
from vllm.model_executor.layers.quantization import get_quantization_config
|
||||
from vllm.model_executor.model_loader.default_loader import DefaultModelLoader
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
def quant_spec(**kwargs: Any) -> QuantSpec:
|
||||
"""Build a `QuantSpec` from user-facing names such as `"mxfp8"`."""
|
||||
return TypeAdapter(QuantSpec).validate_python(kwargs)
|
||||
|
||||
|
||||
def quant_config_args(**kwargs: Any) -> QuantizationConfigArgs:
|
||||
"""Build `QuantizationConfigArgs` from user-facing shorthand values."""
|
||||
return TypeAdapter(QuantizationConfigArgs).validate_python(kwargs)
|
||||
|
||||
|
||||
def _limit_num_hidden_layers(
|
||||
model: torch.nn.Module, num_hidden_layers: int | None
|
||||
) -> None:
|
||||
@@ -59,7 +72,9 @@ def load_model_without_vllm_runner(
|
||||
**(model_config_kwargs or {}),
|
||||
)
|
||||
vllm_config_args = dict(vllm_config_kwargs or {})
|
||||
vllm_config_args.setdefault("compilation_config", CompilationConfig(mode=0))
|
||||
vllm_config_args.setdefault(
|
||||
"compilation_config", CompilationConfig(mode=CompilationMode.NONE)
|
||||
)
|
||||
vllm_config = VllmConfig(model_config=model_config, **vllm_config_args)
|
||||
hf_overrides = (model_config_kwargs or {}).get("hf_overrides") or {}
|
||||
num_hidden_layers = hf_overrides.get("num_hidden_layers")
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
from transformers import AutoTokenizer
|
||||
from transformers import AutoTokenizer, PreTrainedTokenizerBase
|
||||
|
||||
from tests.reasoning.utils import run_reasoning_extraction
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
@@ -39,8 +39,10 @@ REASONING_MODEL_NAME = "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def test_tokenizer():
|
||||
tokenizer = AutoTokenizer.from_pretrained(REASONING_MODEL_NAME)
|
||||
def test_tokenizer() -> PreTrainedTokenizerBase:
|
||||
tokenizer: PreTrainedTokenizerBase = AutoTokenizer.from_pretrained(
|
||||
REASONING_MODEL_NAME
|
||||
)
|
||||
# Add custom test tokens
|
||||
test_tokens = ["<test:think>", "</test:think>", "<alt:start>", "<alt:end>"]
|
||||
existing_tokens = set(tokenizer.get_vocab().keys())
|
||||
@@ -66,7 +68,7 @@ class TestBaseThinkingReasoningParserInit:
|
||||
def test_initialization_with_missing_tokenizer(self):
|
||||
"""Test that initialization fails without tokenizer."""
|
||||
with pytest.raises(ValueError, match="model tokenizer must be passed"):
|
||||
TestThinkingReasoningParser(None)
|
||||
TestThinkingReasoningParser(None) # type: ignore[arg-type]
|
||||
|
||||
def test_initialization_with_missing_tokens(self, test_tokenizer):
|
||||
"""Test that initialization fails when tokens are not in vocabulary."""
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
from transformers import AutoTokenizer
|
||||
from transformers import AutoTokenizer, PreTrainedTokenizerBase
|
||||
|
||||
from tests.reasoning.utils import run_reasoning_extraction
|
||||
from vllm.reasoning import ReasoningParser, ReasoningParserManager
|
||||
@@ -15,7 +15,7 @@ REASONING_MODEL_NAME = "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def deepseek_r1_qwen_tokenizer():
|
||||
def deepseek_r1_qwen_tokenizer() -> PreTrainedTokenizerBase:
|
||||
return AutoTokenizer.from_pretrained(REASONING_MODEL_NAME)
|
||||
|
||||
|
||||
@@ -279,10 +279,10 @@ def test_reasoning(
|
||||
|
||||
# Test extract_content
|
||||
if param_dict["content"] is not None:
|
||||
content = parser.extract_content_ids(output_ids)
|
||||
assert content == deepseek_r1_qwen_tokenizer.convert_tokens_to_ids(
|
||||
content_ids = parser.extract_content_ids(output_ids)
|
||||
assert content_ids == deepseek_r1_qwen_tokenizer.convert_tokens_to_ids(
|
||||
deepseek_r1_qwen_tokenizer.tokenize(param_dict["content"])
|
||||
)
|
||||
else:
|
||||
content = parser.extract_content_ids(output)
|
||||
assert content == []
|
||||
content_ids = parser.extract_content_ids(output)
|
||||
assert content_ids == []
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
from transformers import AutoTokenizer
|
||||
from transformers import AutoTokenizer, PreTrainedTokenizerBase
|
||||
|
||||
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
@@ -17,7 +17,7 @@ REASONING_MODEL_NAME = "deepseek-ai/DeepSeek-V3.1"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def tokenizer():
|
||||
def tokenizer() -> PreTrainedTokenizerBase:
|
||||
return AutoTokenizer.from_pretrained(REASONING_MODEL_NAME)
|
||||
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
from transformers import AutoTokenizer
|
||||
from transformers import AutoTokenizer, PreTrainedTokenizerBase
|
||||
|
||||
from tests.reasoning.utils import run_reasoning_extraction
|
||||
from vllm.reasoning import ReasoningParser, ReasoningParserManager
|
||||
@@ -13,7 +13,7 @@ REASONING_MODEL_NAME = "baidu/ERNIE-4.5-21B-A3B-Thinking"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def ernie45_tokenizer():
|
||||
def ernie45_tokenizer() -> PreTrainedTokenizerBase:
|
||||
return AutoTokenizer.from_pretrained(REASONING_MODEL_NAME)
|
||||
|
||||
|
||||
|
||||
@@ -4,9 +4,11 @@
|
||||
import copy
|
||||
|
||||
import pytest
|
||||
from transformers import AutoTokenizer
|
||||
from transformers import AutoTokenizer, PreTrainedTokenizerBase
|
||||
|
||||
from tests.reasoning.utils import run_reasoning_extraction
|
||||
from vllm.parser.engine.adapters import ParserEngineReasoningAdapter
|
||||
from vllm.parser.glm47_moe import Glm47MoeParser
|
||||
from vllm.reasoning import ReasoningParser, ReasoningParserManager
|
||||
|
||||
parser_name = "glm45"
|
||||
@@ -17,7 +19,7 @@ REASONING_MODEL_NAME = "zai-org/GLM-4.7"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def glm45_tokenizer():
|
||||
def glm45_tokenizer() -> PreTrainedTokenizerBase:
|
||||
return AutoTokenizer.from_pretrained(REASONING_MODEL_NAME)
|
||||
|
||||
|
||||
@@ -261,13 +263,20 @@ def glm53_style_tokenizer(glm45_tokenizer):
|
||||
return tokenizer
|
||||
|
||||
|
||||
def _glm_engine(parser: ReasoningParser) -> Glm47MoeParser:
|
||||
assert isinstance(parser, ParserEngineReasoningAdapter)
|
||||
engine = parser._parser_engine
|
||||
assert isinstance(engine, Glm47MoeParser)
|
||||
return engine
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"disable_kwargs", [{"enable_thinking": False}, {"thinking": False}]
|
||||
)
|
||||
def test_glm53_template_forces_reasoning(disable_kwargs: dict, glm53_style_tokenizer):
|
||||
parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name)
|
||||
parser = parser_cls(glm53_style_tokenizer, chat_template_kwargs=disable_kwargs)
|
||||
assert parser._parser_engine.thinking_enabled
|
||||
assert _glm_engine(parser).thinking_enabled
|
||||
|
||||
output = glm53_style_tokenizer.tokenize(GLM53_LEAK["output"])
|
||||
output_tokens: list[str] = [
|
||||
@@ -298,7 +307,7 @@ def test_glm47_template_honors_thinking_disable(glm45_tokenizer):
|
||||
parser = ReasoningParserManager.get_reasoning_parser(parser_name)(
|
||||
glm45_tokenizer, chat_template_kwargs={"enable_thinking": False}
|
||||
)
|
||||
assert not parser._parser_engine.thinking_enabled
|
||||
assert not _glm_engine(parser).thinking_enabled
|
||||
|
||||
output = glm45_tokenizer.tokenize(GLM53_LEAK["output"])
|
||||
output_tokens: list[str] = [
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import TypedDict
|
||||
from typing import TypedDict, cast
|
||||
|
||||
import pytest
|
||||
import regex as re
|
||||
from transformers import PreTrainedTokenizerBase
|
||||
|
||||
from tests.reasoning.utils import run_reasoning_extraction
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
@@ -52,8 +53,8 @@ class GraniteThinkingTokenizer:
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def tokenizer():
|
||||
return GraniteThinkingTokenizer()
|
||||
def tokenizer() -> PreTrainedTokenizerBase:
|
||||
return cast(PreTrainedTokenizerBase, GraniteThinkingTokenizer())
|
||||
|
||||
|
||||
# ── Basic reasoning extraction (non-streaming + streaming) ───────────
|
||||
@@ -173,7 +174,7 @@ def tokenizer():
|
||||
],
|
||||
)
|
||||
def test_granite_thinking_reasoning(
|
||||
tokenizer: GraniteThinkingTokenizer,
|
||||
tokenizer: PreTrainedTokenizerBase,
|
||||
streaming: bool,
|
||||
param_dict: ReasoningCase,
|
||||
):
|
||||
@@ -195,7 +196,7 @@ def test_granite_thinking_reasoning(
|
||||
|
||||
|
||||
def test_granite_thinking_no_content_after_end_token(
|
||||
tokenizer: GraniteThinkingTokenizer,
|
||||
tokenizer: PreTrainedTokenizerBase,
|
||||
):
|
||||
parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name)
|
||||
parser = parser_cls(tokenizer)
|
||||
@@ -215,7 +216,7 @@ def test_granite_thinking_no_content_after_end_token(
|
||||
|
||||
@pytest.mark.parametrize("streaming", [False, True])
|
||||
def test_granite_thinking_whitespace_only_content(
|
||||
tokenizer: GraniteThinkingTokenizer,
|
||||
tokenizer: PreTrainedTokenizerBase,
|
||||
streaming: bool,
|
||||
):
|
||||
parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name)
|
||||
@@ -235,7 +236,7 @@ def test_granite_thinking_whitespace_only_content(
|
||||
|
||||
|
||||
def test_granite_thinking_unterminated_think_block(
|
||||
tokenizer: GraniteThinkingTokenizer,
|
||||
tokenizer: PreTrainedTokenizerBase,
|
||||
):
|
||||
parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name)
|
||||
parser = parser_cls(tokenizer)
|
||||
@@ -260,7 +261,7 @@ def test_granite_thinking_unterminated_think_block(
|
||||
|
||||
|
||||
def test_granite_thinking_disabled_moves_into_content(
|
||||
tokenizer: GraniteThinkingTokenizer,
|
||||
tokenizer: PreTrainedTokenizerBase,
|
||||
):
|
||||
parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name)
|
||||
parser = parser_cls(tokenizer)
|
||||
@@ -285,7 +286,7 @@ def test_granite_thinking_disabled_moves_into_content(
|
||||
|
||||
|
||||
def test_granite_thinking_disabled_with_leading_newline(
|
||||
tokenizer: GraniteThinkingTokenizer,
|
||||
tokenizer: PreTrainedTokenizerBase,
|
||||
):
|
||||
# With enable_thinking=False, model output goes through the swap
|
||||
# path: content is initially None (all text classified as
|
||||
@@ -316,7 +317,7 @@ def test_granite_thinking_disabled_with_leading_newline(
|
||||
|
||||
|
||||
def test_granite_thinking_force_nonempty_content_moves_into_content(
|
||||
tokenizer: GraniteThinkingTokenizer,
|
||||
tokenizer: PreTrainedTokenizerBase,
|
||||
):
|
||||
parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name)
|
||||
parser = parser_cls(tokenizer)
|
||||
@@ -338,7 +339,7 @@ def test_granite_thinking_force_nonempty_content_moves_into_content(
|
||||
|
||||
|
||||
def test_granite_thinking_force_nonempty_no_swap_when_newlines_only(
|
||||
tokenizer: GraniteThinkingTokenizer,
|
||||
tokenizer: PreTrainedTokenizerBase,
|
||||
):
|
||||
# When </think> IS present and content is newlines-only, lstrip
|
||||
# removes them but the swap should NOT fire — content was present,
|
||||
@@ -363,7 +364,7 @@ def test_granite_thinking_force_nonempty_no_swap_when_newlines_only(
|
||||
|
||||
|
||||
def test_granite_thinking_force_nonempty_swaps_when_content_absent(
|
||||
tokenizer: GraniteThinkingTokenizer,
|
||||
tokenizer: PreTrainedTokenizerBase,
|
||||
):
|
||||
# When </think> IS present but content is truly absent (zero
|
||||
# characters after </think>, e.g. max_tokens cut), swap fires.
|
||||
@@ -387,7 +388,7 @@ def test_granite_thinking_force_nonempty_swaps_when_content_absent(
|
||||
|
||||
|
||||
def test_granite_thinking_force_nonempty_keeps_real_content(
|
||||
tokenizer: GraniteThinkingTokenizer,
|
||||
tokenizer: PreTrainedTokenizerBase,
|
||||
):
|
||||
parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name)
|
||||
parser = parser_cls(tokenizer)
|
||||
@@ -412,7 +413,7 @@ def test_granite_thinking_force_nonempty_keeps_real_content(
|
||||
|
||||
|
||||
def test_granite_thinking_keeps_truncated_reasoning(
|
||||
tokenizer: GraniteThinkingTokenizer,
|
||||
tokenizer: PreTrainedTokenizerBase,
|
||||
):
|
||||
parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name)
|
||||
parser = parser_cls(tokenizer)
|
||||
@@ -472,7 +473,7 @@ def _run_parse_delta(parser, tokenizer, text, request):
|
||||
|
||||
|
||||
def test_granite_thinking_streaming_enable_thinking_false(
|
||||
tokenizer: GraniteThinkingTokenizer,
|
||||
tokenizer: PreTrainedTokenizerBase,
|
||||
):
|
||||
# With enable_thinking=False, the parser (constructed without
|
||||
# kwargs) starts in REASONING state. All text streams as reasoning
|
||||
@@ -494,7 +495,7 @@ def test_granite_thinking_streaming_enable_thinking_false(
|
||||
|
||||
|
||||
def test_granite_thinking_streaming_strips_leading_newline(
|
||||
tokenizer: GraniteThinkingTokenizer,
|
||||
tokenizer: PreTrainedTokenizerBase,
|
||||
):
|
||||
request = ChatCompletionRequest(
|
||||
model="test-model",
|
||||
@@ -511,7 +512,7 @@ def test_granite_thinking_streaming_strips_leading_newline(
|
||||
|
||||
|
||||
def test_granite_thinking_streaming_promotes_reasoning_to_content(
|
||||
tokenizer: GraniteThinkingTokenizer,
|
||||
tokenizer: PreTrainedTokenizerBase,
|
||||
):
|
||||
request = ChatCompletionRequest(
|
||||
model="test-model",
|
||||
@@ -527,7 +528,7 @@ def test_granite_thinking_streaming_promotes_reasoning_to_content(
|
||||
|
||||
|
||||
def test_granite_thinking_streaming_no_promotion_with_real_content(
|
||||
tokenizer: GraniteThinkingTokenizer,
|
||||
tokenizer: PreTrainedTokenizerBase,
|
||||
):
|
||||
request = ChatCompletionRequest(
|
||||
model="test-model",
|
||||
@@ -548,7 +549,7 @@ def test_granite_thinking_streaming_no_promotion_with_real_content(
|
||||
|
||||
|
||||
def test_granite_thinking_streaming_no_promotion_without_opt_in(
|
||||
tokenizer: GraniteThinkingTokenizer,
|
||||
tokenizer: PreTrainedTokenizerBase,
|
||||
):
|
||||
request = ChatCompletionRequest(model="test-model", messages=[])
|
||||
parser = _make_reasoning_parser(tokenizer)
|
||||
@@ -564,7 +565,7 @@ def test_granite_thinking_streaming_no_promotion_without_opt_in(
|
||||
|
||||
@pytest.mark.parametrize("streaming", [False, True])
|
||||
def test_granite_thinking_empty_think_block(
|
||||
tokenizer: GraniteThinkingTokenizer,
|
||||
tokenizer: PreTrainedTokenizerBase,
|
||||
streaming: bool,
|
||||
):
|
||||
# <think></think>\nHello — empty reasoning, content after newline.
|
||||
@@ -592,7 +593,7 @@ def test_granite_thinking_empty_think_block(
|
||||
],
|
||||
)
|
||||
def test_granite_thinking_streaming_chunking_independent(
|
||||
tokenizer: GraniteThinkingTokenizer,
|
||||
tokenizer: PreTrainedTokenizerBase,
|
||||
text: str,
|
||||
expected_reasoning: str,
|
||||
expected_content: str,
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
from transformers import AutoTokenizer
|
||||
from transformers import AutoTokenizer, PreTrainedTokenizerBase
|
||||
|
||||
from tests.reasoning.utils import run_reasoning_extraction
|
||||
from vllm.reasoning import ReasoningParser, ReasoningParserManager
|
||||
@@ -15,7 +15,7 @@ REASONING_MODEL_NAME = "HCompany/Holo2-4B"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def tokenizer():
|
||||
def tokenizer() -> PreTrainedTokenizerBase:
|
||||
return AutoTokenizer.from_pretrained(REASONING_MODEL_NAME)
|
||||
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
import string
|
||||
from collections.abc import Sequence
|
||||
from typing import cast
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -10,6 +11,7 @@ from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionReque
|
||||
from vllm.parser.parser_manager import ParserManager
|
||||
from vllm.reasoning import ReasoningParserManager
|
||||
from vllm.reasoning.k2_horizon_reasoning_parser import K2HorizonReasoningParser
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
|
||||
@@ -59,6 +61,10 @@ class K2Tokenizer:
|
||||
return "".join(inverse_vocab[token_id] for token_id in ids)
|
||||
|
||||
|
||||
def _k2_tokenizer() -> TokenizerLike:
|
||||
return cast(TokenizerLike, K2Tokenizer())
|
||||
|
||||
|
||||
def _request(**kwargs) -> ChatCompletionRequest:
|
||||
return ChatCompletionRequest(
|
||||
model="k2-horizon",
|
||||
@@ -70,7 +76,7 @@ def _request(**kwargs) -> ChatCompletionRequest:
|
||||
@pytest.mark.parametrize("effort", ["high", "medium", "low"])
|
||||
def test_reasoning_effort_selects_tokens(effort: str):
|
||||
parser = K2HorizonReasoningParser(
|
||||
K2Tokenizer(),
|
||||
_k2_tokenizer(),
|
||||
chat_template_kwargs={"reasoning_effort": effort},
|
||||
)
|
||||
|
||||
@@ -78,7 +84,7 @@ def test_reasoning_effort_selects_tokens(effort: str):
|
||||
|
||||
|
||||
def test_reasoning_effort_precedence_and_default():
|
||||
tokenizer = K2Tokenizer()
|
||||
tokenizer = _k2_tokenizer()
|
||||
request = _request(
|
||||
reasoning_effort="medium",
|
||||
chat_template_kwargs={"reasoning_effort": "low"},
|
||||
@@ -103,13 +109,13 @@ def test_reasoning_effort_precedence_and_default():
|
||||
def test_invalid_reasoning_effort_raises(effort: str | None):
|
||||
with pytest.raises(ValueError, match="Unsupported reasoning_effort"):
|
||||
K2HorizonReasoningParser(
|
||||
K2Tokenizer(),
|
||||
_k2_tokenizer(),
|
||||
chat_template_kwargs={"reasoning_effort": effort},
|
||||
)
|
||||
|
||||
|
||||
def test_non_streaming_explicit_and_implicit_boundaries():
|
||||
parser = K2HorizonReasoningParser(K2Tokenizer())
|
||||
parser = K2HorizonReasoningParser(_k2_tokenizer())
|
||||
request = _request()
|
||||
|
||||
assert parser.extract_reasoning("<ifm|think>plan</ifm|think>answer", request) == (
|
||||
@@ -126,7 +132,7 @@ def test_non_streaming_explicit_and_implicit_boundaries():
|
||||
|
||||
@pytest.mark.parametrize("reasoning", ["plan", " ", "\n", " \n"])
|
||||
def test_character_split_reasoning_stream(reasoning: str):
|
||||
parser = K2HorizonReasoningParser(K2Tokenizer())
|
||||
parser = K2HorizonReasoningParser(_k2_tokenizer())
|
||||
reasoning_parts: list[str] = []
|
||||
content_parts: list[str] = []
|
||||
|
||||
@@ -150,7 +156,7 @@ def test_character_split_reasoning_stream(reasoning: str):
|
||||
|
||||
|
||||
def test_latest_boundary_controls_reasoning_state():
|
||||
tokenizer = K2Tokenizer()
|
||||
tokenizer = _k2_tokenizer()
|
||||
parser = K2HorizonReasoningParser(tokenizer)
|
||||
vocab = tokenizer.get_vocab()
|
||||
|
||||
@@ -159,7 +165,7 @@ def test_latest_boundary_controls_reasoning_state():
|
||||
|
||||
|
||||
def test_composed_streaming_reasoning_to_tool_handoff():
|
||||
tokenizer = K2Tokenizer()
|
||||
tokenizer = _k2_tokenizer()
|
||||
tools = [
|
||||
{
|
||||
"type": "function",
|
||||
|
||||
@@ -6,6 +6,7 @@ import pytest
|
||||
|
||||
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.parser.kimi_k2 import KimiK2Parser
|
||||
from vllm.reasoning.kimi_k2_reasoning_parser import KimiK2ReasoningParser
|
||||
from vllm.tokenizers import get_tokenizer
|
||||
|
||||
@@ -65,9 +66,13 @@ def test_extract_reasoning_tool_section_ends_reasoning(kimi_k2_tokenizer):
|
||||
def test_streaming_reasoning_then_content(kimi_k2_tokenizer):
|
||||
"""Token-by-token streaming: reasoning tokens then content after </think>."""
|
||||
parser = KimiK2ReasoningParser(kimi_k2_tokenizer)
|
||||
parser_engine = parser._parser_engine
|
||||
assert isinstance(parser_engine, KimiK2Parser)
|
||||
|
||||
think_id = parser._parser_engine._start_token_id
|
||||
end_think_id = parser._parser_engine._end_token_id
|
||||
think_id = parser_engine._start_token_id
|
||||
end_think_id = parser_engine._end_token_id
|
||||
assert think_id is not None
|
||||
assert end_think_id is not None
|
||||
# Use a real token ID from the tokenizer for regular content
|
||||
regular_id = kimi_k2_tokenizer.encode("hello", add_special_tokens=False)[0]
|
||||
|
||||
@@ -123,9 +128,13 @@ def test_streaming_reasoning_then_content(kimi_k2_tokenizer):
|
||||
def test_streaming_tool_section_ends_reasoning(kimi_k2_tokenizer):
|
||||
"""<|tool_calls_section_begin|> in delta ends reasoning during streaming."""
|
||||
parser = KimiK2ReasoningParser(kimi_k2_tokenizer)
|
||||
parser_engine = parser._parser_engine
|
||||
assert isinstance(parser_engine, KimiK2Parser)
|
||||
|
||||
think_id = parser._parser_engine._start_token_id
|
||||
tool_begin_id = parser._parser_engine._tool_section_start_token_id
|
||||
think_id = parser_engine._start_token_id
|
||||
tool_begin_id = parser_engine._tool_section_start_token_id
|
||||
assert think_id is not None
|
||||
assert tool_begin_id is not None
|
||||
regular_id = kimi_k2_tokenizer.encode("hello", add_special_tokens=False)[0]
|
||||
|
||||
# Tool section token arrives — should transition from reasoning to content
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import cast
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||
@@ -10,6 +12,7 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
from vllm.parser.kimi_k3 import KimiK3Parser
|
||||
from vllm.parser.parser_manager import ParserManager
|
||||
from vllm.reasoning.kimi_k3_reasoning_parser import KimiK3ReasoningParser
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
|
||||
@@ -25,10 +28,13 @@ RESPONSE_OPEN_IDS = [ord(ch) for ch in RESPONSE_OPEN]
|
||||
|
||||
|
||||
class DummyTokenizer:
|
||||
def __len__(self) -> int:
|
||||
return 1
|
||||
|
||||
def get_vocab(self) -> dict[str, int]:
|
||||
return {}
|
||||
|
||||
def encode(self, text: str, add_special_tokens: bool = False) -> list[int]:
|
||||
def encode(self, text: str, *args, **kwargs) -> list[int]:
|
||||
if text == THINK_OPEN:
|
||||
return [1, 2, 3]
|
||||
if text == THINK_CLOSE:
|
||||
@@ -36,6 +42,10 @@ class DummyTokenizer:
|
||||
return [ord(ch) for ch in text]
|
||||
|
||||
|
||||
def _dummy_tokenizer() -> TokenizerLike:
|
||||
return cast(TokenizerLike, DummyTokenizer())
|
||||
|
||||
|
||||
class ReasoningOnlyParser(KimiK3Parser):
|
||||
reasoning_parser_cls = KimiK3ReasoningParser
|
||||
|
||||
@@ -84,7 +94,7 @@ def test_extract_reasoning_with_generation_prefix_consumed():
|
||||
|
||||
|
||||
def test_delegating_parser_strips_response_wrapper_without_tool_parser():
|
||||
parser = ReasoningOnlyParser(DummyTokenizer())
|
||||
parser = ReasoningOnlyParser(_dummy_tokenizer())
|
||||
request = ChatCompletionRequest(model="test-model", messages=[])
|
||||
|
||||
reasoning, content, tool_calls = parser.parse(
|
||||
@@ -165,7 +175,7 @@ def test_count_reasoning_tokens_is_zero_when_thinking_disabled():
|
||||
|
||||
|
||||
def test_count_reasoning_tokens_through_delegating_parser():
|
||||
parser = ReasoningOnlyParser(DummyTokenizer())
|
||||
parser = ReasoningOnlyParser(_dummy_tokenizer())
|
||||
|
||||
assert parser.count_reasoning_tokens([*OPEN_IDS, 9, *CLOSE_IDS, 11]) == 1
|
||||
|
||||
@@ -253,7 +263,8 @@ def test_thinking_disabled_streams_content():
|
||||
|
||||
def test_delegating_parser_thinking_false_streams_response_content():
|
||||
parser = ReasoningOnlyParser(
|
||||
DummyTokenizer(), chat_template_kwargs={"thinking": False}
|
||||
_dummy_tokenizer(),
|
||||
chat_template_kwargs={"thinking": False},
|
||||
)
|
||||
request = ChatCompletionRequest(
|
||||
model="test-model",
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
from transformers import AutoTokenizer
|
||||
from transformers import AutoTokenizer, PreTrainedTokenizerBase
|
||||
|
||||
from tests.reasoning.utils import run_reasoning_extraction
|
||||
from vllm.reasoning import ReasoningParser, ReasoningParserManager
|
||||
@@ -15,7 +15,7 @@ REASONING_MODEL_NAME = "MiniMaxAI/MiniMax-M2"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def minimax_m2_tokenizer():
|
||||
def minimax_m2_tokenizer() -> PreTrainedTokenizerBase:
|
||||
return AutoTokenizer.from_pretrained(REASONING_MODEL_NAME)
|
||||
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
from transformers import AutoTokenizer
|
||||
from transformers import AutoTokenizer, PreTrainedTokenizerBase
|
||||
|
||||
from tests.reasoning.utils import run_reasoning_extraction
|
||||
from vllm.reasoning import ReasoningParser, ReasoningParserManager
|
||||
@@ -15,7 +15,7 @@ REASONING_MODEL_NAME = "MiniMaxAI/MiniMax-M2"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def minimax_m2_tokenizer():
|
||||
def minimax_m2_tokenizer() -> PreTrainedTokenizerBase:
|
||||
return AutoTokenizer.from_pretrained(REASONING_MODEL_NAME)
|
||||
|
||||
|
||||
@@ -195,10 +195,10 @@ def test_reasoning(
|
||||
|
||||
# Test extract_content
|
||||
if param_dict["content"] is not None:
|
||||
content = parser.extract_content_ids(output_ids)
|
||||
assert content == minimax_m2_tokenizer.convert_tokens_to_ids(
|
||||
content_ids = parser.extract_content_ids(output_ids)
|
||||
assert content_ids == minimax_m2_tokenizer.convert_tokens_to_ids(
|
||||
minimax_m2_tokenizer.tokenize(param_dict["content"])
|
||||
)
|
||||
else:
|
||||
content = parser.extract_content_ids(output)
|
||||
assert content == []
|
||||
content_ids = parser.extract_content_ids(output)
|
||||
assert content_ids == []
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import TypedDict
|
||||
from typing import TypedDict, cast
|
||||
|
||||
import pytest
|
||||
import regex as re
|
||||
from transformers import PreTrainedTokenizerBase
|
||||
|
||||
from tests.reasoning.utils import run_reasoning_extraction
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
@@ -48,8 +49,8 @@ class FakeNemotronTokenizer:
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def tokenizer():
|
||||
return FakeNemotronTokenizer()
|
||||
def tokenizer() -> PreTrainedTokenizerBase:
|
||||
return cast(PreTrainedTokenizerBase, FakeNemotronTokenizer())
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
@@ -94,7 +95,7 @@ def tokenizer():
|
||||
],
|
||||
)
|
||||
def test_nemotron_v3_reasoning(
|
||||
tokenizer: FakeNemotronTokenizer,
|
||||
tokenizer: PreTrainedTokenizerBase,
|
||||
streaming: bool,
|
||||
param_dict: ReasoningCase,
|
||||
):
|
||||
@@ -113,7 +114,7 @@ def test_nemotron_v3_reasoning(
|
||||
|
||||
|
||||
def test_nemotron_v3_without_thinking_moves_into_content(
|
||||
tokenizer: FakeNemotronTokenizer,
|
||||
tokenizer: PreTrainedTokenizerBase,
|
||||
):
|
||||
parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name)
|
||||
parser = parser_cls(tokenizer)
|
||||
@@ -137,7 +138,7 @@ def test_nemotron_v3_without_thinking_moves_into_content(
|
||||
|
||||
|
||||
def test_nemotron_v3_force_nonempty_content_moves_into_content(
|
||||
tokenizer: FakeNemotronTokenizer,
|
||||
tokenizer: PreTrainedTokenizerBase,
|
||||
):
|
||||
parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name)
|
||||
parser = parser_cls(tokenizer)
|
||||
@@ -159,7 +160,7 @@ def test_nemotron_v3_force_nonempty_content_moves_into_content(
|
||||
|
||||
|
||||
def test_nemotron_v3_force_nonempty_keeps_real_content(
|
||||
tokenizer: FakeNemotronTokenizer,
|
||||
tokenizer: PreTrainedTokenizerBase,
|
||||
):
|
||||
# When real content follows the closing tag nothing is promoted: the
|
||||
# content after </think> is returned as-is and reasoning stays separate.
|
||||
@@ -183,7 +184,7 @@ def test_nemotron_v3_force_nonempty_keeps_real_content(
|
||||
|
||||
|
||||
def test_nemotron_v3_with_thinking_keeps_truncated_reasoning(
|
||||
tokenizer: FakeNemotronTokenizer,
|
||||
tokenizer: PreTrainedTokenizerBase,
|
||||
):
|
||||
parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name)
|
||||
parser = parser_cls(tokenizer)
|
||||
@@ -242,7 +243,7 @@ def _run_parse_delta(parser, tokenizer, text, request):
|
||||
|
||||
|
||||
def test_nemotron_v3_streaming_promotes_reasoning_to_content(
|
||||
tokenizer: FakeNemotronTokenizer,
|
||||
tokenizer: PreTrainedTokenizerBase,
|
||||
):
|
||||
# Model never closes <think>: reasoning streams normally AND is duplicated
|
||||
# into content on the terminal delta.
|
||||
@@ -260,7 +261,7 @@ def test_nemotron_v3_streaming_promotes_reasoning_to_content(
|
||||
|
||||
|
||||
def test_nemotron_v3_streaming_no_promotion_with_real_content(
|
||||
tokenizer: FakeNemotronTokenizer,
|
||||
tokenizer: PreTrainedTokenizerBase,
|
||||
):
|
||||
request = ChatCompletionRequest(
|
||||
model="test-model",
|
||||
@@ -279,7 +280,7 @@ def test_nemotron_v3_streaming_no_promotion_with_real_content(
|
||||
|
||||
|
||||
def test_nemotron_v3_streaming_no_promotion_without_opt_in(
|
||||
tokenizer: FakeNemotronTokenizer,
|
||||
tokenizer: PreTrainedTokenizerBase,
|
||||
):
|
||||
# Without enable_thinking=False / force_nonempty_content the fallback must
|
||||
# stay disabled: the response stays reasoning-only, content empty.
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
from transformers import AutoTokenizer
|
||||
from transformers import AutoTokenizer, PreTrainedTokenizerBase
|
||||
|
||||
from tests.reasoning.utils import (
|
||||
StreamingReasoningReconstructor,
|
||||
@@ -24,7 +24,7 @@ REASONING_MODEL_NAMES = [
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", params=REASONING_MODEL_NAMES)
|
||||
def qwen3_tokenizer(request):
|
||||
def qwen3_tokenizer(request) -> PreTrainedTokenizerBase:
|
||||
return AutoTokenizer.from_pretrained(request.param)
|
||||
|
||||
|
||||
|
||||
@@ -4,7 +4,8 @@
|
||||
import pytest
|
||||
|
||||
from tests.reasoning.utils import run_reasoning_extraction
|
||||
from vllm.reasoning import ReasoningParser, ReasoningParserManager
|
||||
from vllm.reasoning import ReasoningParserManager
|
||||
from vllm.reasoning.step3p5_reasoning_parser import Step3p5ReasoningParser
|
||||
from vllm.tokenizers import get_tokenizer
|
||||
|
||||
parser_name = "step3p5"
|
||||
@@ -283,9 +284,8 @@ def test_reasoning(
|
||||
output_tokens: list[str] = [
|
||||
step3p5_tokenizer.convert_tokens_to_string([token]) for token in output
|
||||
]
|
||||
parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser(parser_name)(
|
||||
step3p5_tokenizer
|
||||
)
|
||||
parser = ReasoningParserManager.get_reasoning_parser(parser_name)(step3p5_tokenizer)
|
||||
assert isinstance(parser, Step3p5ReasoningParser)
|
||||
|
||||
reasoning, content = run_reasoning_extraction(
|
||||
parser, output_tokens, streaming=streaming
|
||||
@@ -317,7 +317,7 @@ def test_reasoning(
|
||||
|
||||
# Test extract_content
|
||||
if param_dict["content"] is not None:
|
||||
content = parser.extract_content_ids(output_ids)
|
||||
content_ids = parser.extract_content_ids(output_ids)
|
||||
# Fixed expected token ids for specific test cases
|
||||
test_id = (
|
||||
request.node.callspec.id if hasattr(request.node, "callspec") else None
|
||||
@@ -332,7 +332,7 @@ def test_reasoning(
|
||||
expected_content_ids = step3p5_tokenizer.convert_tokens_to_ids(
|
||||
step3p5_tokenizer.tokenize(param_dict["content"])
|
||||
)
|
||||
assert content == expected_content_ids
|
||||
assert content_ids == expected_content_ids
|
||||
else:
|
||||
content = parser.extract_content_ids(output_ids)
|
||||
assert content == []
|
||||
content_ids = parser.extract_content_ids(output_ids)
|
||||
assert content_ids == []
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import cast
|
||||
|
||||
from transformers import PreTrainedTokenizerBase
|
||||
|
||||
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.reasoning import ReasoningParser
|
||||
@@ -102,10 +106,11 @@ def run_reasoning_extraction_streaming(
|
||||
reconstructor = StreamingReasoningReconstructor()
|
||||
previous_text = ""
|
||||
previous_tokens: list[int] = []
|
||||
tokenizer = cast(PreTrainedTokenizerBase, reasoning_parser.model_tokenizer)
|
||||
for delta in model_deltas:
|
||||
token_delta = [
|
||||
reasoning_parser.vocab.get(token)
|
||||
for token in reasoning_parser.model_tokenizer.tokenize(delta)
|
||||
reasoning_parser.vocab[token]
|
||||
for token in tokenizer.tokenize(delta)
|
||||
if token in reasoning_parser.vocab
|
||||
]
|
||||
current_text = previous_text + delta
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
|
||||
import random
|
||||
from collections.abc import Callable
|
||||
from typing import NamedTuple, TypeAlias
|
||||
from typing import NamedTuple, NoReturn, TypeAlias
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
@@ -13,6 +13,7 @@ from tests.utils import create_new_process_for_each_test
|
||||
from tests.v1.sample.utils import (
|
||||
LogitsprocsTestFakes,
|
||||
create_fake_logits,
|
||||
create_mock_reasoning_config,
|
||||
create_penalty_tensor,
|
||||
create_prompt_tokens_tensor,
|
||||
fake_apply_logitsprocs,
|
||||
@@ -103,14 +104,6 @@ class LogitsProcsRequestParams:
|
||||
return f"MyClass({summ})"
|
||||
|
||||
|
||||
class MockReasoningConfig:
|
||||
"""Minimal reasoning config for ``ThinkingBudgetStateHolder`` tests."""
|
||||
|
||||
reasoning_start_token_ids = [THINK_START_TOKEN_ID]
|
||||
reasoning_end_token_ids = [THINK_END_TOKEN_ID]
|
||||
enabled = True
|
||||
|
||||
|
||||
def _generate_fake_sampling_metadata(
|
||||
num_output_tokens: int,
|
||||
batch_size: int,
|
||||
@@ -131,7 +124,9 @@ def _generate_fake_sampling_metadata(
|
||||
)
|
||||
|
||||
vllm_config = VllmConfig()
|
||||
vllm_config.reasoning_config = MockReasoningConfig()
|
||||
vllm_config.reasoning_config = create_mock_reasoning_config(
|
||||
[THINK_START_TOKEN_ID], [THINK_END_TOKEN_ID]
|
||||
)
|
||||
|
||||
logitsprocs = build_logitsprocs(
|
||||
vllm_config=vllm_config,
|
||||
@@ -242,7 +237,7 @@ def _raise_error_invalid(
|
||||
request_params: LogitsProcsRequestParams,
|
||||
step_idx: int,
|
||||
err_cls: type[Exception] = ValueError,
|
||||
) -> None:
|
||||
) -> NoReturn:
|
||||
raise err_cls(
|
||||
f"Validation failed for step={step_idx}, "
|
||||
f"batch_index={batch_index}, "
|
||||
@@ -268,6 +263,7 @@ def _logit_bias_validate(
|
||||
) -> None:
|
||||
"""Validate logit bias logitproc applied correctly."""
|
||||
logit_bias = request_params.params.logit_bias
|
||||
assert logit_bias is not None
|
||||
logits_old = test_fakes.logits[persistent_batch[batch_index].workload_index].cpu()
|
||||
logits_new = logits_new[batch_index].cpu()
|
||||
for token_id in range(VOCAB_SIZE):
|
||||
@@ -368,9 +364,7 @@ def _min_tokens_validate(
|
||||
ref_num_out_tokens = len(request_params.out_tokens)
|
||||
min_reached = ref_num_out_tokens >= MIN_TOKENS_LEN_THRESHOLD
|
||||
ref_all_stop_token_ids = request_params.params.all_stop_token_ids
|
||||
mt_lp: MinTokensLogitsProcessor = next(
|
||||
test_fakes.get_logitsprocs_by_cls(MinTokensLogitsProcessor)
|
||||
)
|
||||
mt_lp = next(test_fakes.get_logitsprocs_by_cls(MinTokensLogitsProcessor))
|
||||
assert isinstance(mt_lp, MinTokensLogitsProcessor)
|
||||
min_tok = mt_lp.min_toks.get(batch_index, None)
|
||||
|
||||
@@ -689,7 +683,7 @@ class LogitsprocTestHelpers(NamedTuple):
|
||||
gen_request_fxn: Callable | None = None
|
||||
|
||||
|
||||
logitsprocs_test_mapping = {
|
||||
logitsprocs_test_mapping: dict[LogitprocType, LogitsprocTestHelpers] = {
|
||||
STR_NO_LOGITPROC: LogitsprocTestHelpers(eval_fxn=_none_validate),
|
||||
LogitBiasLogitsProcessor: LogitsprocTestHelpers(
|
||||
gen_request_fxn=_logit_bias_params, eval_fxn=_logit_bias_validate
|
||||
@@ -706,22 +700,22 @@ logitsprocs_test_mapping = {
|
||||
}
|
||||
|
||||
|
||||
def _get_test_cases() -> list[list[str]]:
|
||||
def _get_test_cases() -> list[list[LogitprocType]]:
|
||||
"""Each test case is a set of logitsprocs."""
|
||||
logitsprocs_types = list(logitsprocs_test_mapping.keys())
|
||||
logitsprocs_types: list[LogitprocType] = list(logitsprocs_test_mapping.keys())
|
||||
|
||||
# Isolate thinking-budget handling from other processors to avoid cross-talk.
|
||||
thinking_id: LogitprocType = STR_THINKING_BUDGET
|
||||
other_processors = [
|
||||
other_processors: list[LogitprocType] = [
|
||||
p for p in logitsprocs_types if p != STR_NO_LOGITPROC and p != thinking_id
|
||||
]
|
||||
|
||||
return (
|
||||
[[STR_NO_LOGITPROC]]
|
||||
+ [[logitproc_type, STR_NO_LOGITPROC] for logitproc_type in other_processors]
|
||||
+ [other_processors]
|
||||
+ [[thinking_id]]
|
||||
test_cases: list[list[LogitprocType]] = [[STR_NO_LOGITPROC]]
|
||||
test_cases.extend(
|
||||
[logitproc_type, STR_NO_LOGITPROC] for logitproc_type in other_processors
|
||||
)
|
||||
test_cases.extend([other_processors, [thinking_id]])
|
||||
return test_cases
|
||||
|
||||
|
||||
def _generate_fake_step_update(
|
||||
@@ -774,6 +768,7 @@ def _generate_fake_step_update(
|
||||
for add_req_params in workload_params[wdx : (wdx + num_step_add_replace)]:
|
||||
# Replace as many removed requests as possible with added requests
|
||||
add_remove_idx = batch_update_builder.pop_removed()
|
||||
assert add_remove_idx is not None
|
||||
batch_update_builder.added.append(
|
||||
(
|
||||
add_remove_idx,
|
||||
@@ -976,13 +971,6 @@ def test_logitsprocs(
|
||||
step_idx += 1
|
||||
|
||||
|
||||
class MockReasoningNoEndTokens:
|
||||
"""Reasoning config with no end token ids (disables enforcement in holder)."""
|
||||
|
||||
reasoning_start_token_ids = [THINK_START_TOKEN_ID]
|
||||
reasoning_end_token_ids: list[int] = []
|
||||
|
||||
|
||||
def test_maybe_create_thinking_budget_holder_without_reasoning():
|
||||
cfg = VllmConfig()
|
||||
assert cfg.reasoning_config is None
|
||||
@@ -1000,7 +988,9 @@ def test_maybe_create_thinking_budget_holder_without_reasoning():
|
||||
|
||||
def test_thinking_budget_holder_has_tracked_after_sync_add():
|
||||
vc = VllmConfig()
|
||||
vc.reasoning_config = MockReasoningConfig()
|
||||
vc.reasoning_config = create_mock_reasoning_config(
|
||||
[THINK_START_TOKEN_ID], [THINK_END_TOKEN_ID]
|
||||
)
|
||||
h = ThinkingBudgetStateHolder(
|
||||
vc.reasoning_config,
|
||||
vc.scheduler_config.max_num_seqs,
|
||||
@@ -1030,7 +1020,9 @@ def test_thinking_budget_holder_has_tracked_after_sync_add():
|
||||
|
||||
def test_thinking_budget_holder_sync_remove_clears_state():
|
||||
vc = VllmConfig()
|
||||
vc.reasoning_config = MockReasoningConfig()
|
||||
vc.reasoning_config = create_mock_reasoning_config(
|
||||
[THINK_START_TOKEN_ID], [THINK_END_TOKEN_ID]
|
||||
)
|
||||
h = ThinkingBudgetStateHolder(
|
||||
vc.reasoning_config,
|
||||
vc.scheduler_config.max_num_seqs,
|
||||
@@ -1060,7 +1052,9 @@ def test_thinking_budget_holder_sync_remove_clears_state():
|
||||
|
||||
def test_thinking_budget_holder_sync_add_without_budget_drops_row():
|
||||
vc = VllmConfig()
|
||||
vc.reasoning_config = MockReasoningConfig()
|
||||
vc.reasoning_config = create_mock_reasoning_config(
|
||||
[THINK_START_TOKEN_ID], [THINK_END_TOKEN_ID]
|
||||
)
|
||||
h = ThinkingBudgetStateHolder(
|
||||
vc.reasoning_config,
|
||||
vc.scheduler_config.max_num_seqs,
|
||||
@@ -1081,7 +1075,9 @@ def test_thinking_budget_holder_sync_add_without_budget_drops_row():
|
||||
|
||||
def test_thinking_budget_holder_swap_exchanges_state():
|
||||
vc = VllmConfig()
|
||||
vc.reasoning_config = MockReasoningConfig()
|
||||
vc.reasoning_config = create_mock_reasoning_config(
|
||||
[THINK_START_TOKEN_ID], [THINK_END_TOKEN_ID]
|
||||
)
|
||||
h = ThinkingBudgetStateHolder(
|
||||
vc.reasoning_config,
|
||||
vc.scheduler_config.max_num_seqs,
|
||||
@@ -1125,7 +1121,9 @@ def test_thinking_budget_holder_swap_exchanges_state():
|
||||
|
||||
def test_thinking_budget_holder_unidirectional_move():
|
||||
vc = VllmConfig()
|
||||
vc.reasoning_config = MockReasoningConfig()
|
||||
vc.reasoning_config = create_mock_reasoning_config(
|
||||
[THINK_START_TOKEN_ID], [THINK_END_TOKEN_ID]
|
||||
)
|
||||
h = ThinkingBudgetStateHolder(
|
||||
vc.reasoning_config,
|
||||
vc.scheduler_config.max_num_seqs,
|
||||
@@ -1163,7 +1161,9 @@ def test_thinking_budget_holder_unidirectional_move():
|
||||
|
||||
def test_thinking_budget_holder_update_state_repeat_indices_last_row_wins():
|
||||
vc = VllmConfig()
|
||||
vc.reasoning_config = MockReasoningConfig()
|
||||
vc.reasoning_config = create_mock_reasoning_config(
|
||||
[THINK_START_TOKEN_ID], [THINK_END_TOKEN_ID]
|
||||
)
|
||||
h = ThinkingBudgetStateHolder(
|
||||
vc.reasoning_config,
|
||||
vc.scheduler_config.max_num_seqs,
|
||||
@@ -1197,7 +1197,7 @@ def test_thinking_budget_holder_update_state_repeat_indices_last_row_wins():
|
||||
|
||||
def test_thinking_budget_holder_spec_mode_tensor_layout():
|
||||
h = ThinkingBudgetStateHolder(
|
||||
MockReasoningConfig(),
|
||||
create_mock_reasoning_config([THINK_START_TOKEN_ID], [THINK_END_TOKEN_ID]),
|
||||
8,
|
||||
2,
|
||||
torch.device("cpu"),
|
||||
@@ -1209,7 +1209,7 @@ def test_thinking_budget_holder_spec_mode_tensor_layout():
|
||||
|
||||
def test_thinking_budget_holder_empty_end_tokens_disables_row():
|
||||
vc = VllmConfig()
|
||||
vc.reasoning_config = MockReasoningNoEndTokens()
|
||||
vc.reasoning_config = create_mock_reasoning_config([THINK_START_TOKEN_ID], [])
|
||||
h = ThinkingBudgetStateHolder(
|
||||
vc.reasoning_config,
|
||||
vc.scheduler_config.max_num_seqs,
|
||||
@@ -1253,7 +1253,9 @@ def test_thinking_budget_enforced_without_penalties():
|
||||
passing an empty list (the pre-fix behavior) prevents budget enforcement.
|
||||
"""
|
||||
vc = VllmConfig()
|
||||
vc.reasoning_config = MockReasoningConfig()
|
||||
vc.reasoning_config = create_mock_reasoning_config(
|
||||
[THINK_START_TOKEN_ID], [THINK_END_TOKEN_ID]
|
||||
)
|
||||
budget = 3 # allow 3 thinking tokens
|
||||
|
||||
h = ThinkingBudgetStateHolder(
|
||||
@@ -1346,7 +1348,11 @@ def test_thinking_budget_long_thinking_section_end_marker_found_at_correct_index
|
||||
"""Test thinking budget enforced for a long thinking run,
|
||||
then a natural end marker."""
|
||||
h = ThinkingBudgetStateHolder(
|
||||
MockReasoningConfig(), 8, 0, torch.device("cpu"), False
|
||||
create_mock_reasoning_config([THINK_START_TOKEN_ID], [THINK_END_TOKEN_ID]),
|
||||
8,
|
||||
0,
|
||||
torch.device("cpu"),
|
||||
False,
|
||||
)
|
||||
h.sync_batch(
|
||||
BatchUpdate(
|
||||
@@ -1356,8 +1362,8 @@ def test_thinking_budget_long_thinking_section_end_marker_found_at_correct_index
|
||||
moved=(),
|
||||
)
|
||||
)
|
||||
start = MockReasoningConfig.reasoning_start_token_ids
|
||||
end = MockReasoningConfig.reasoning_end_token_ids
|
||||
start = [THINK_START_TOKEN_ID]
|
||||
end = [THINK_END_TOKEN_ID]
|
||||
|
||||
out: list[int] = list(start)
|
||||
h.update_state([out], None, None)
|
||||
@@ -1396,13 +1402,9 @@ class TestThinkingBudgetReentry:
|
||||
|
||||
@staticmethod
|
||||
def _make_holder(end_token_ids: list[int]) -> ThinkingBudgetStateHolder:
|
||||
class FakeReasoningConfig:
|
||||
reasoning_start_token_ids = [TestThinkingBudgetReentry.THINK_START]
|
||||
reasoning_end_token_ids: list[int] = []
|
||||
enabled = True
|
||||
|
||||
cfg = FakeReasoningConfig()
|
||||
cfg.reasoning_end_token_ids = end_token_ids
|
||||
cfg = create_mock_reasoning_config(
|
||||
[TestThinkingBudgetReentry.THINK_START], end_token_ids
|
||||
)
|
||||
return ThinkingBudgetStateHolder(
|
||||
reasoning_config=cfg,
|
||||
max_num_seqs=8,
|
||||
@@ -1526,21 +1528,12 @@ class TestThinkingBudgetNaturalEndReentry:
|
||||
|
||||
@staticmethod
|
||||
def _make_holder(end_token_ids):
|
||||
from dataclasses import dataclass
|
||||
|
||||
from vllm.v1.sample.thinking_budget_state import (
|
||||
ThinkingBudgetStateHolder,
|
||||
)
|
||||
|
||||
@dataclass
|
||||
class FakeReasoningConfig:
|
||||
reasoning_start_token_ids: list[int]
|
||||
reasoning_end_token_ids: list[int]
|
||||
enabled: bool = True
|
||||
|
||||
cfg = FakeReasoningConfig(
|
||||
reasoning_start_token_ids=[TestThinkingBudgetNaturalEndReentry.THINK_START],
|
||||
reasoning_end_token_ids=end_token_ids,
|
||||
cfg = create_mock_reasoning_config(
|
||||
[TestThinkingBudgetNaturalEndReentry.THINK_START], end_token_ids
|
||||
)
|
||||
return ThinkingBudgetStateHolder(
|
||||
reasoning_config=cfg,
|
||||
@@ -1830,21 +1823,11 @@ class TestThinkingBudgetNaturalEndReentry:
|
||||
natural </think> exit, continue_thinking is cleared and Block 2
|
||||
enforcement works correctly.
|
||||
"""
|
||||
from dataclasses import dataclass
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from vllm.v1.sample.thinking_budget_state import ThinkingBudgetStateHolder
|
||||
|
||||
@dataclass
|
||||
class FakeReasoningConfig:
|
||||
reasoning_start_token_ids: list[int]
|
||||
reasoning_end_token_ids: list[int]
|
||||
enabled: bool = True
|
||||
|
||||
cfg = FakeReasoningConfig(
|
||||
reasoning_start_token_ids=[self.THINK_START],
|
||||
reasoning_end_token_ids=self.THINK_END_SINGLE,
|
||||
)
|
||||
cfg = create_mock_reasoning_config([self.THINK_START], self.THINK_END_SINGLE)
|
||||
holder = ThinkingBudgetStateHolder(
|
||||
reasoning_config=cfg,
|
||||
max_num_seqs=8,
|
||||
|
||||
@@ -57,15 +57,20 @@ prompts = [
|
||||
class DummyLogitsProcessor(LogitsProcessor):
|
||||
"""Fake logit processor to support unit testing and examples."""
|
||||
|
||||
@classmethod
|
||||
def validate_params(cls, params: SamplingParams):
|
||||
target_token: int | None = params.extra_args and params.extra_args.get(
|
||||
"target_token"
|
||||
@staticmethod
|
||||
def _get_target_token(params: SamplingParams) -> int | None:
|
||||
target_token = (
|
||||
params.extra_args.get(DUMMY_LOGITPROC_ARG) if params.extra_args else None
|
||||
)
|
||||
if target_token is not None and not isinstance(target_token, int):
|
||||
raise VLLMValidationError(
|
||||
f"target_token value {target_token} {type(target_token)} is not int"
|
||||
)
|
||||
return target_token
|
||||
|
||||
@classmethod
|
||||
def validate_params(cls, params: SamplingParams):
|
||||
cls._get_target_token(params)
|
||||
|
||||
def __init__(
|
||||
self, vllm_config: "VllmConfig", device: torch.device, is_pin_memory: bool
|
||||
@@ -78,8 +83,7 @@ class DummyLogitsProcessor(LogitsProcessor):
|
||||
|
||||
def update_state(self, batch_update: BatchUpdate | None):
|
||||
def extract_extra_arg(params: SamplingParams) -> int | None:
|
||||
self.validate_params(params)
|
||||
return params.extra_args and params.extra_args.get("target_token")
|
||||
return self._get_target_token(params)
|
||||
|
||||
process_dict_updates(
|
||||
self.req_info,
|
||||
|
||||
@@ -341,6 +341,7 @@ def test_batch_allowed_needs_one_shared_set():
|
||||
assert states.batch_allowed([]) is None
|
||||
|
||||
shared = states.batch_allowed([0])
|
||||
assert shared is not None
|
||||
assert shared.tolist() == [1, 2, 3]
|
||||
assert shared.dtype == torch.int64
|
||||
assert states.batch_allowed([0, 0]) is shared
|
||||
|
||||
@@ -34,6 +34,8 @@ def test_stats_match_reference(vocab: int, dtype: torch.dtype):
|
||||
)
|
||||
assert torch.equal(argmax, ref_argmax)
|
||||
torch.testing.assert_close(entropy, ref_entropy, atol=2e-3, rtol=1e-3)
|
||||
assert probs is not None
|
||||
assert ref_probs is not None
|
||||
torch.testing.assert_close(probs.float(), ref_probs.float(), atol=5e-3, rtol=2e-2)
|
||||
assert probs.dtype == torch.bfloat16
|
||||
assert probs.shape == logits.shape
|
||||
@@ -100,6 +102,7 @@ def test_masked_logits_keep_a_finite_entropy():
|
||||
assert entropy[0].item() < 1e-6 and entropy[1].item() < 1e-6
|
||||
assert abs(entropy[2].item() - math.log(2.0)) < 1e-4
|
||||
assert torch.equal(argmax[[0, 1, 3]], torch.tensor([3, 3, 3], device="cuda"))
|
||||
assert probs is not None
|
||||
assert probs[0, 3].item() == 1.0 and probs[0].sum().item() == 1.0
|
||||
_, _, ref_entropy, _ = sample_row_stats_reference(logits, temps, 1, None)
|
||||
assert torch.isfinite(ref_entropy).all()
|
||||
|
||||
@@ -340,7 +340,7 @@ def _make_v2_state(
|
||||
device=device,
|
||||
all_token_ids=SimpleNamespace(gpu=all_tokens),
|
||||
)
|
||||
state = DryState(_make_vllm_config(), req_states)
|
||||
state = DryState(_make_vllm_config(), req_states) # type: ignore[arg-type]
|
||||
if tokenizer is not None:
|
||||
# Re-resolve the default set now that a tokenizer is installed.
|
||||
state._tokenizer = tokenizer
|
||||
@@ -446,7 +446,7 @@ def test_spec_decode_is_refused_at_construction():
|
||||
)
|
||||
config = _make_vllm_config(speculative_config=SimpleNamespace())
|
||||
with pytest.raises(ValueError, match="speculative decoding"):
|
||||
DryState(config, req_states)
|
||||
DryState(config, req_states) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_v2_draft_expanded_logits_raise():
|
||||
@@ -917,7 +917,10 @@ def test_loads_through_the_loader_by_fqcn():
|
||||
total_len=None,
|
||||
)
|
||||
procs = loader.build_custom_logits_processors(
|
||||
_make_vllm_config(), req_states, False, [DRY_FQCN]
|
||||
_make_vllm_config(),
|
||||
req_states, # type: ignore[arg-type]
|
||||
False,
|
||||
[DRY_FQCN],
|
||||
)
|
||||
assert [type(p) for p in procs] == [DryState]
|
||||
|
||||
|
||||
@@ -16,6 +16,7 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
UnquantizedEmbeddingMethod,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
|
||||
|
||||
@@ -32,10 +33,17 @@ class _FakeLmHead:
|
||||
self.tp_size = 1
|
||||
|
||||
|
||||
def _fake_lm_head(
|
||||
weight: torch.Tensor, quantized: bool = False, shard_indices: object | None = None
|
||||
) -> VocabParallelEmbedding:
|
||||
# The dtype tests only need the projection fields, not distributed setup.
|
||||
return _FakeLmHead(weight, quantized, shard_indices) # type: ignore[return-value]
|
||||
|
||||
|
||||
def _build_processor(vocab_size: int) -> LogitsProcessor:
|
||||
lp = LogitsProcessor(vocab_size)
|
||||
# The TP gather is orthogonal to the dtype behavior under test.
|
||||
lp._gather_logits = lambda logits: logits
|
||||
lp._gather_logits = lambda logits: logits # type: ignore[method-assign]
|
||||
return lp
|
||||
|
||||
|
||||
@@ -47,8 +55,9 @@ def test_fp32_head_runs_projection_in_fp32(default_vllm_config):
|
||||
hidden_states = torch.randn(num_tokens, hidden_size, dtype=torch.bfloat16)
|
||||
weight = torch.randn(vocab_size, hidden_size, dtype=torch.bfloat16)
|
||||
|
||||
logits = lp._get_logits(hidden_states, _FakeLmHead(weight), None)
|
||||
logits = lp._get_logits(hidden_states, _fake_lm_head(weight), None)
|
||||
|
||||
assert logits is not None
|
||||
assert logits.dtype == torch.float32
|
||||
assert torch.isfinite(logits).all()
|
||||
expected = torch.nn.functional.linear(hidden_states.float(), weight.float())
|
||||
@@ -66,8 +75,9 @@ def test_non_fp32_head_dtype_uses_cast_path(default_vllm_config):
|
||||
hidden_states = torch.randn(4, hidden_size, dtype=torch.bfloat16)
|
||||
weight = torch.randn(vocab_size, hidden_size, dtype=torch.bfloat16)
|
||||
|
||||
logits = lp._get_logits(hidden_states, _FakeLmHead(weight), None)
|
||||
logits = lp._get_logits(hidden_states, _fake_lm_head(weight), None)
|
||||
|
||||
assert logits is not None
|
||||
assert logits.dtype == torch.float16
|
||||
expected = torch.nn.functional.linear(hidden_states.half(), weight.half())
|
||||
torch.testing.assert_close(logits, expected)
|
||||
@@ -82,7 +92,7 @@ def test_head_dtype_equal_to_model_dtype_uses_quant_method(default_vllm_config):
|
||||
|
||||
hidden_states = torch.randn(4, hidden_size, dtype=torch.bfloat16)
|
||||
weight = torch.randn(vocab_size, hidden_size, dtype=torch.bfloat16)
|
||||
lm_head = _FakeLmHead(weight)
|
||||
lm_head = _fake_lm_head(weight)
|
||||
|
||||
with mock.patch.object(
|
||||
lm_head.quant_method,
|
||||
@@ -94,6 +104,7 @@ def test_head_dtype_equal_to_model_dtype_uses_quant_method(default_vllm_config):
|
||||
logits = lp._get_logits(hidden_states, lm_head, None)
|
||||
|
||||
apply_mock.assert_called_once()
|
||||
assert logits is not None
|
||||
assert logits.dtype == torch.bfloat16
|
||||
|
||||
|
||||
@@ -120,9 +131,10 @@ def test_fp32_head_uses_mm_fast_path_on_device(default_vllm_config):
|
||||
with mock.patch(
|
||||
"vllm.model_executor.layers.logits_processor.F.linear"
|
||||
) as linear_mock:
|
||||
logits = lp._get_logits(hidden_states, _FakeLmHead(weight), None)
|
||||
logits = lp._get_logits(hidden_states, _fake_lm_head(weight), None)
|
||||
|
||||
linear_mock.assert_not_called()
|
||||
assert logits is not None
|
||||
assert logits.dtype == torch.float32
|
||||
expected = torch.nn.functional.linear(hidden_states.float(), weight.float())
|
||||
torch.testing.assert_close(logits, expected)
|
||||
@@ -131,7 +143,7 @@ def test_fp32_head_uses_mm_fast_path_on_device(default_vllm_config):
|
||||
def test_fp32_head_rejects_quantized_lm_head(default_vllm_config):
|
||||
lp = _build_processor(64)
|
||||
lp.head_dtype = torch.float32
|
||||
lm_head = _FakeLmHead(torch.randn(64, 16, dtype=torch.bfloat16), quantized=True)
|
||||
lm_head = _fake_lm_head(torch.randn(64, 16, dtype=torch.bfloat16), quantized=True)
|
||||
|
||||
with pytest.raises(ValueError, match="unquantized"):
|
||||
lp._get_logits(torch.randn(4, 16, dtype=torch.bfloat16), lm_head, None)
|
||||
@@ -197,7 +209,7 @@ def test_get_top_tokens_honors_head_dtype(default_vllm_config):
|
||||
|
||||
hidden_states = torch.randn(4, hidden_size, dtype=torch.bfloat16)
|
||||
weight = torch.randn(vocab_size, hidden_size, dtype=torch.bfloat16)
|
||||
lm_head = _FakeLmHead(
|
||||
lm_head = _fake_lm_head(
|
||||
weight,
|
||||
shard_indices=types.SimpleNamespace(
|
||||
num_org_vocab_padding=0, org_vocab_start_index=0
|
||||
@@ -236,7 +248,9 @@ def test_fp32_head_e2e_no_nan():
|
||||
|
||||
for output in outputs:
|
||||
for completion in output.outputs:
|
||||
assert completion.logprobs is not None
|
||||
for token_id, position in zip(completion.token_ids, completion.logprobs):
|
||||
assert position is not None
|
||||
# The sampled token survived filtering, so its logprob is finite.
|
||||
assert math.isfinite(position[token_id].logprob)
|
||||
# No returned logprob is NaN.
|
||||
|
||||
@@ -24,6 +24,7 @@ from vllm import SamplingParams
|
||||
from vllm.config.model import LogprobsMode
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.logprobs import Logprob
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.engine.input_processor import InputProcessor
|
||||
|
||||
@@ -145,7 +146,7 @@ def _repeat_logprob_config(
|
||||
def _run_and_validate(
|
||||
vllm_model: VllmRunner,
|
||||
test_prompts: list[str],
|
||||
vllm_sampling_params: SamplingParams,
|
||||
vllm_sampling_params: SamplingParams | list[SamplingParams],
|
||||
hf_logprobs: list[list[torch.Tensor]],
|
||||
hf_outputs: list[tuple[list[int], str]],
|
||||
logprob_prompt_logprob_list: BatchLogprobsSpecType,
|
||||
@@ -163,11 +164,12 @@ def _run_and_validate(
|
||||
# Extract request-level (prompt)logprobs config
|
||||
num_top_logprobs, num_top_prompt_logprobs = logprob_prompt_logprob
|
||||
|
||||
assert vllm_result.prompt_token_ids is not None
|
||||
# Test whether sampled token output is consistent between vLLM and HF
|
||||
# vLLM prompt+completion should match HF output
|
||||
if temperature == 0.0:
|
||||
assert (
|
||||
vllm_result.prompt_token_ids + vllm_result.outputs[0].token_ids
|
||||
vllm_result.prompt_token_ids + list(vllm_result.outputs[0].token_ids)
|
||||
== hf_output[0]
|
||||
)
|
||||
else:
|
||||
@@ -191,7 +193,9 @@ def _run_and_validate(
|
||||
|
||||
# Confirm that the output token appears among the logprobs
|
||||
assert token_id in logprobs
|
||||
token_in_topk = logprobs[token_id].rank <= num_top_logprobs
|
||||
rank = logprobs[token_id].rank
|
||||
assert rank is not None
|
||||
token_in_topk = rank <= num_top_logprobs
|
||||
|
||||
# If the output token is not included in the top K
|
||||
# logprob, it can return 1 more data
|
||||
@@ -207,8 +211,10 @@ def _run_and_validate(
|
||||
|
||||
output_text = vllm_result.outputs[0].text
|
||||
output_string_from_most_likely_tokens_lst: list[str] = []
|
||||
for top_logprobs in vllm_result.outputs[0].logprobs:
|
||||
top_logprob = next(iter(top_logprobs.values()))
|
||||
for output_logprobs in vllm_result.outputs[0].logprobs:
|
||||
assert output_logprobs is not None
|
||||
top_logprob = next(iter(output_logprobs.values()))
|
||||
assert top_logprob.decoded_token is not None
|
||||
output_string_from_most_likely_tokens_lst.append(
|
||||
top_logprob.decoded_token
|
||||
)
|
||||
@@ -226,8 +232,9 @@ def _run_and_validate(
|
||||
|
||||
# Compare vLLM sample logprobs to HF
|
||||
vllm_sample_logprobs = vllm_result.outputs[0].logprobs
|
||||
for i, top_logprobs in enumerate(vllm_sample_logprobs):
|
||||
for token_id, sample_logprob in top_logprobs.items():
|
||||
for i, sample_logprobs in enumerate(vllm_sample_logprobs):
|
||||
assert sample_logprobs is not None
|
||||
for token_id, sample_logprob in sample_logprobs.items():
|
||||
if temperature == 0.0 or i == 0:
|
||||
logprob = sample_logprob.logprob
|
||||
torch.testing.assert_close(
|
||||
@@ -271,9 +278,9 @@ def _run_and_validate(
|
||||
|
||||
# Confirm that the prompt token appears among the logprobs
|
||||
assert prompt_token_id in prompt_logprobs
|
||||
token_in_topk = (
|
||||
prompt_logprobs[prompt_token_id].rank <= num_top_prompt_logprobs
|
||||
)
|
||||
prompt_rank = prompt_logprobs[prompt_token_id].rank
|
||||
assert prompt_rank is not None
|
||||
token_in_topk = prompt_rank <= num_top_prompt_logprobs
|
||||
|
||||
# If the prompt token is not included in the top K
|
||||
# logprob, it can return 1 more data
|
||||
@@ -294,9 +301,10 @@ def _run_and_validate(
|
||||
# 1:.
|
||||
vllm_prompt_logprobs = vllm_result.prompt_logprobs[1:]
|
||||
for i, vllm_prompt_logprob_dict in enumerate(vllm_prompt_logprobs):
|
||||
for token_id, logprob in vllm_prompt_logprob_dict.items():
|
||||
assert vllm_prompt_logprob_dict is not None
|
||||
for token_id, prompt_lp in vllm_prompt_logprob_dict.items():
|
||||
torch.testing.assert_close(
|
||||
logprob.logprob,
|
||||
prompt_lp.logprob,
|
||||
hf_logprob[0][i][token_id].item(),
|
||||
atol=2e-2,
|
||||
rtol=2e-2,
|
||||
@@ -478,12 +486,12 @@ def test_prompt_logprob_token_ids_require_v2_model_runner():
|
||||
params = SamplingParams(prompt_logprob_token_ids=[1, 2])
|
||||
with patch.object(SamplingParams, "verify"):
|
||||
with pytest.raises(VLLMValidationError, match="V2 model runner"):
|
||||
InputProcessor._validate_params(processor, params, ("generate",))
|
||||
InputProcessor._validate_params(processor, params, ("generate",)) # type: ignore[arg-type]
|
||||
processor.vllm_config.use_v2_model_runner = True
|
||||
InputProcessor._validate_params(processor, params, ("generate",))
|
||||
InputProcessor._validate_params(processor, params, ("generate",)) # type: ignore[arg-type]
|
||||
processor.vllm_config.cache_config.kv_sharing_fast_prefill = True
|
||||
with pytest.raises(VLLMValidationError, match="fast-prefill"):
|
||||
InputProcessor._validate_params(processor, params, ("generate",))
|
||||
InputProcessor._validate_params(processor, params, ("generate",)) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_none_logprobs(vllm_model, example_prompts):
|
||||
@@ -607,7 +615,9 @@ def test_logprobs_mode(logprobs_mode: LogprobsMode):
|
||||
total_token_with_logprobs = 0
|
||||
positive_values = 0
|
||||
for output in results[0].outputs:
|
||||
assert output.logprobs is not None
|
||||
for logprobs in output.logprobs:
|
||||
assert logprobs is not None
|
||||
for token_id in logprobs:
|
||||
logprob = logprobs[token_id]
|
||||
if logprobs_mode in ("raw_logprobs", "processed_logprobs"):
|
||||
@@ -648,6 +658,7 @@ def test_prompt_logprobs_mode():
|
||||
)
|
||||
assert results[0].prompt_logprobs is not None
|
||||
assert results[0].prompt_logprobs[1] is not None
|
||||
assert results[0].prompt_token_ids is not None
|
||||
tok_id = results[0].prompt_token_ids[1]
|
||||
values[mode] = results[0].prompt_logprobs[1][tok_id].logprob
|
||||
finally:
|
||||
@@ -1228,10 +1239,12 @@ def test_spec_decode_logprobs(
|
||||
[prompt, prompt], [sampling_params, penalty_sampling_params]
|
||||
)
|
||||
# Collect logprobs outputs from reference LLM.
|
||||
ref_logprobs = []
|
||||
ref_logprobs: list[Logprob] = []
|
||||
for results in ref_results:
|
||||
for output in results.outputs:
|
||||
assert output.logprobs is not None
|
||||
for logprobs in output.logprobs:
|
||||
assert logprobs is not None
|
||||
ref_logprobs.extend(logprobs.values())
|
||||
del ref_llm
|
||||
torch.accelerator.empty_cache()
|
||||
@@ -1249,10 +1262,12 @@ def test_spec_decode_logprobs(
|
||||
[prompt, prompt], [sampling_params, penalty_sampling_params]
|
||||
)
|
||||
# Collect logprobs outputs from spec decode LLM.
|
||||
spec_logprobs = []
|
||||
spec_logprobs: list[Logprob] = []
|
||||
for results in spec_results:
|
||||
for output in results.outputs:
|
||||
assert output.logprobs is not None
|
||||
for logprobs in output.logprobs:
|
||||
assert logprobs is not None
|
||||
spec_logprobs.extend(logprobs.values())
|
||||
del spec_llm
|
||||
torch.accelerator.empty_cache()
|
||||
@@ -1326,6 +1341,7 @@ def test_prompt_logprobs_with_chunking_and_preemption():
|
||||
"Unexpected number of prompt logprob positions"
|
||||
)
|
||||
|
||||
assert sampling_params.prompt_logprobs is not None
|
||||
# Each position should have the requested number of logprobs
|
||||
for pos, logprobs_dict in enumerate(prompt_logprobs):
|
||||
if logprobs_dict is not None: # First token may be None
|
||||
|
||||
@@ -333,6 +333,7 @@ def test_sampler_repetition_penalty(
|
||||
for batch_idx in range(batch_size):
|
||||
non_penalized_token_id = logits[batch_idx].argmax().item()
|
||||
penalized_token_id = logits[batch_idx].argmin().item()
|
||||
assert sampling_metadata.prompt_token_ids is not None
|
||||
prompt_tokens = sampling_metadata.prompt_token_ids[batch_idx][:].tolist()
|
||||
output_tokens = sampling_metadata.output_token_ids[batch_idx]
|
||||
if repetition_penalty > 1.0:
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user