[Mini-vLLM] SM86-only cut: models, backends, Rust, tests

- Keep only Qwen3.5, Qwen3.5-MTP and Qwen4Exp model paths; trim the model registry to 41 entries
- Delete idle attention backends (MLA, FlashInfer, ROCm-only, quant backends) and dead GDN CuTeDSL kernels
- Remove inc/quark quant loaders; keep compressed-tensors and Marlin
- Drop the Rust frontend, pooling entrypoints, and idle test suites
- Rebuild kernels for sm_80;sm_86 and strip the extensions
- Verified: full import sweep clean (optional deps only) and a dummy-load Qwen3.5 engine run on RTX 3090 via the Triton GDN path

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-18 19:19:56 +07:00
co-authored by Hinata
parent c5ab42bc0f
commit e61dcb9143
2728 changed files with 159 additions and 950401 deletions
-375
View File
@@ -1,375 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import tempfile
from pathlib import Path
import pytest
from vllm.benchmarks.sweep import serve as sweep_serve
from vllm.benchmarks.sweep.param_sweep import ParameterSweep, ParameterSweepItem
class TestParameterSweepItem:
"""Test ParameterSweepItem functionality."""
@pytest.mark.parametrize(
"input_dict,expected",
[
(
{"compilation_config.use_inductor_graph_partition": False},
"--compilation-config.use_inductor_graph_partition=false",
),
(
{"compilation_config.use_inductor_graph_partition": True},
"--compilation-config.use_inductor_graph_partition=true",
),
],
)
def test_nested_boolean_params(self, input_dict, expected):
"""Test that nested boolean params use =true/false syntax."""
item = ParameterSweepItem.from_record(input_dict)
cmd = item.apply_to_cmd(["vllm", "serve", "model"])
assert expected in cmd
@pytest.mark.parametrize(
"input_dict,expected",
[
({"enable_prefix_caching": False}, "--no-enable-prefix-caching"),
({"enable_prefix_caching": True}, "--enable-prefix-caching"),
({"disable_log_stats": False}, "--no-disable-log-stats"),
({"disable_log_stats": True}, "--disable-log-stats"),
],
)
def test_non_nested_boolean_params(self, input_dict, expected):
"""Test that non-nested boolean params use --no- prefix."""
item = ParameterSweepItem.from_record(input_dict)
cmd = item.apply_to_cmd(["vllm", "serve", "model"])
assert expected in cmd
@pytest.mark.parametrize(
"compilation_config",
[
{"cudagraph_mode": "full", "mode": 2, "use_inductor_graph_partition": True},
{
"cudagraph_mode": "piecewise",
"mode": 3,
"use_inductor_graph_partition": False,
},
],
)
def test_nested_dict_value(self, compilation_config):
"""Test that nested dict values are serialized as JSON."""
item = ParameterSweepItem.from_record(
{"compilation_config": compilation_config}
)
cmd = item.apply_to_cmd(["vllm", "serve", "model"])
assert "--compilation-config" in cmd
# The dict should be JSON serialized
idx = cmd.index("--compilation-config")
assert json.loads(cmd[idx + 1]) == compilation_config
@pytest.mark.parametrize(
"input_dict,expected_key,expected_value",
[
({"model": "test-model"}, "--model", "test-model"),
({"max_tokens": 100}, "--max-tokens", "100"),
({"temperature": 0.7}, "--temperature", "0.7"),
],
)
def test_string_and_numeric_values(self, input_dict, expected_key, expected_value):
"""Test that string and numeric values are handled correctly."""
item = ParameterSweepItem.from_record(input_dict)
cmd = item.apply_to_cmd(["vllm", "serve"])
assert expected_key in cmd
assert expected_value in cmd
@pytest.mark.parametrize(
"input_dict,expected_key,key_idx_offset",
[
({"max_tokens": 200}, "--max-tokens", 1),
({"enable_prefix_caching": False}, "--no-enable-prefix-caching", 0),
],
)
def test_replace_existing_parameter(self, input_dict, expected_key, key_idx_offset):
"""Test that existing parameters in cmd are replaced."""
item = ParameterSweepItem.from_record(input_dict)
if key_idx_offset == 1:
# Key-value pair
cmd = item.apply_to_cmd(["vllm", "serve", "--max-tokens", "100", "model"])
assert expected_key in cmd
idx = cmd.index(expected_key)
assert cmd[idx + 1] == "200"
assert "100" not in cmd
else:
# Boolean flag
cmd = item.apply_to_cmd(
["vllm", "serve", "--enable-prefix-caching", "model"]
)
assert expected_key in cmd
assert "--enable-prefix-caching" not in cmd
class TestParameterSweep:
"""Test ParameterSweep functionality."""
def test_from_records_list(self):
"""Test creating ParameterSweep from a list of records."""
records: list[dict[str, object]] = [
{"max_tokens": 100, "temperature": 0.7},
{"max_tokens": 200, "temperature": 0.9},
]
sweep = ParameterSweep.from_records(records)
assert len(sweep) == 2
assert sweep[0]["max_tokens"] == 100
assert sweep[1]["max_tokens"] == 200
def test_read_from_dict(self):
"""Test creating ParameterSweep from a dict format."""
data: dict[str, dict[str, object]] = {
"experiment1": {"max_tokens": 100, "temperature": 0.7},
"experiment2": {"max_tokens": 200, "temperature": 0.9},
}
sweep = ParameterSweep.read_from_dict(data)
assert len(sweep) == 2
# Check that items have the _benchmark_name field
names = {item["_benchmark_name"] for item in sweep}
assert names == {"experiment1", "experiment2"}
# Check that parameters are preserved
for item in sweep:
if item["_benchmark_name"] == "experiment1":
assert item["max_tokens"] == 100
assert item["temperature"] == 0.7
elif item["_benchmark_name"] == "experiment2":
assert item["max_tokens"] == 200
assert item["temperature"] == 0.9
def test_read_json_list_format(self):
"""Test reading JSON file with list format."""
records: list[dict[str, object]] = [
{"max_tokens": 100, "temperature": 0.7},
{"max_tokens": 200, "temperature": 0.9},
]
with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f:
json.dump(records, f)
temp_path = Path(f.name)
try:
sweep = ParameterSweep.read_json(temp_path)
assert len(sweep) == 2
assert sweep[0]["max_tokens"] == 100
assert sweep[1]["max_tokens"] == 200
finally:
temp_path.unlink()
def test_read_json_dict_format(self):
"""Test reading JSON file with dict format."""
data: dict[str, dict[str, object]] = {
"experiment1": {"max_tokens": 100, "temperature": 0.7},
"experiment2": {"max_tokens": 200, "temperature": 0.9},
}
with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f:
json.dump(data, f)
temp_path = Path(f.name)
try:
sweep = ParameterSweep.read_json(temp_path)
assert len(sweep) == 2
# Check that items have the _benchmark_name field
names = {item["_benchmark_name"] for item in sweep}
assert names == {"experiment1", "experiment2"}
finally:
temp_path.unlink()
def test_unique_benchmark_names_validation(self):
"""Test that duplicate _benchmark_name values raise an error."""
# Test with duplicate names in list format
records: list[dict[str, object]] = [
{"_benchmark_name": "exp1", "max_tokens": 100},
{"_benchmark_name": "exp1", "max_tokens": 200},
]
with pytest.raises(ValueError, match="Duplicate _benchmark_name values"):
ParameterSweep.from_records(records)
def test_unique_benchmark_names_multiple_duplicates(self):
"""Test validation with multiple duplicate names."""
records: list[dict[str, object]] = [
{"_benchmark_name": "exp1", "max_tokens": 100},
{"_benchmark_name": "exp1", "max_tokens": 200},
{"_benchmark_name": "exp2", "max_tokens": 300},
{"_benchmark_name": "exp2", "max_tokens": 400},
]
with pytest.raises(ValueError, match="Duplicate _benchmark_name values"):
ParameterSweep.from_records(records)
def test_no_benchmark_names_allowed(self):
"""Test that records without _benchmark_name are allowed."""
records: list[dict[str, object]] = [
{"max_tokens": 100, "temperature": 0.7},
{"max_tokens": 200, "temperature": 0.9},
]
sweep = ParameterSweep.from_records(records)
assert len(sweep) == 2
def test_mixed_benchmark_names_allowed(self):
"""Test that mixing records with and without _benchmark_name is allowed."""
records: list[dict[str, object]] = [
{"_benchmark_name": "exp1", "max_tokens": 100},
{"max_tokens": 200, "temperature": 0.9},
]
sweep = ParameterSweep.from_records(records)
assert len(sweep) == 2
class TestParameterSweepItemKeyNormalization:
"""Test key normalization in ParameterSweepItem."""
def test_underscore_to_hyphen_conversion(self):
"""Test that underscores are converted to hyphens in CLI."""
item = ParameterSweepItem.from_record({"max_tokens": 100})
cmd = item.apply_to_cmd(["vllm", "serve"])
assert "--max-tokens" in cmd
def test_nested_key_preserves_suffix(self):
"""Test that nested keys preserve the suffix format."""
# The suffix after the dot should preserve underscores
item = ParameterSweepItem.from_record(
{"compilation_config.some_nested_param": "value"}
)
cmd = item.apply_to_cmd(["vllm", "serve"])
# The prefix (compilation_config) gets converted to hyphens,
# but the suffix (some_nested_param) is preserved
assert any("compilation-config.some_nested_param" in arg for arg in cmd)
def test_run_comb_excludes_warmup_from_measured_results(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
):
calls: list[tuple[int, int, str]] = []
def fake_run_benchmark(
server,
bench_cmd,
*,
serve_overrides,
bench_overrides,
run_number,
output_path,
dry_run,
):
calls.append(
(run_number, int(bench_overrides["num_prompts"]), output_path.name)
)
return {"run_number": run_number}
monkeypatch.setattr(sweep_serve, "run_benchmark", fake_run_benchmark)
base_path = tmp_path / "combination"
base_path.mkdir()
measured = sweep_serve.run_comb(
None,
[],
serve_comb=ParameterSweepItem(),
bench_comb=ParameterSweepItem({"num_prompts": 320}),
link_vars=[],
base_path=base_path,
num_runs=2,
warmup_num_prompts=32,
dry_run=False,
)
assert calls == [
(-1, 32, "warmup.json"),
(0, 320, "run=0.json"),
(1, 320, "run=1.json"),
]
assert measured == [{"run_number": 0}, {"run_number": 1}]
def test_run_comb_continue_on_error_keeps_later_runs(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
):
calls: list[int] = []
def fake_run_benchmark(
server,
bench_cmd,
*,
serve_overrides,
bench_overrides,
run_number,
output_path,
dry_run,
):
calls.append(run_number)
if run_number == 0:
raise RuntimeError("synthetic run failure")
return {"run_number": run_number}
monkeypatch.setattr(sweep_serve, "run_benchmark", fake_run_benchmark)
base_path = tmp_path / "combination"
base_path.mkdir()
measured = sweep_serve.run_comb(
None,
[],
serve_comb=ParameterSweepItem(),
bench_comb=ParameterSweepItem({"num_prompts": 100}),
link_vars=[],
base_path=base_path,
num_runs=2,
warmup_num_prompts=0,
dry_run=False,
continue_on_error=True,
)
assert calls == [0, 1]
assert measured == [{"run_number": 1}]
assert (base_path / "run=0.failure.json").exists()
assert (base_path / "summary.json").exists()
def test_run_comb_warmup_default_is_backward_compatible(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
):
"""Older callers can omit warmup_num_prompts."""
calls: list[int] = []
def fake_run_benchmark(
server,
bench_cmd,
*,
serve_overrides,
bench_overrides,
run_number,
output_path,
dry_run,
):
calls.append(run_number)
return {"run_number": run_number}
monkeypatch.setattr(sweep_serve, "run_benchmark", fake_run_benchmark)
base_path = tmp_path / "combination"
base_path.mkdir()
measured = sweep_serve.run_comb(
None,
[],
serve_comb=ParameterSweepItem(),
bench_comb=ParameterSweepItem({"num_prompts": 10}),
link_vars=[],
base_path=base_path,
num_runs=1,
dry_run=False,
)
assert calls == [0]
assert measured == [{"run_number": 0}]
-330
View File
@@ -1,330 +0,0 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
from pathlib import Path
from typing import Protocol, cast
import numpy as np
import pytest
import soundfile as sf
import vllm.benchmarks.datasets.datasets as datasets_module
import vllm.benchmarks.lib.endpoint_request_func as request_func_module
from vllm.benchmarks.lib.endpoint_request_func import RequestFuncInput
from vllm.tokenizers import TokenizerLike
pytestmark = pytest.mark.skip_global_cleanup
class _ReadableBinary(Protocol):
def read(self, size: int = -1) -> bytes: ...
class _TokenizedPrompt:
def __init__(self, prompt: str) -> None:
self.input_ids = prompt.split()
class _Tokenizer:
def __init__(self, name_or_path: str = "openai/whisper-large-v3") -> None:
self.name_or_path = name_or_path
def __call__(self, prompt: str) -> _TokenizedPrompt:
return _TokenizedPrompt(prompt)
class CohereAsrTokenizer(_Tokenizer):
def __init__(self, name_or_path: str = "/models/cohere-transcribe") -> None:
super().__init__(name_or_path)
class _CohereNameOnlyTokenizer(_Tokenizer):
def __init__(self) -> None:
super().__init__("cohere/some-local-checkpoint")
def _as_tokenizer(tokenizer: _Tokenizer) -> TokenizerLike:
return cast(TokenizerLike, tokenizer)
def _write_wav(path: Path, duration_s: float = 0.1, sample_rate: int = 16_000) -> None:
num_samples = int(duration_s * sample_rate)
sf.write(path, np.zeros(num_samples, dtype=np.float32), sample_rate)
class _FakeFormData:
def __init__(self) -> None:
self.fields: list[tuple[str, object, dict[str, str]]] = []
def add_field(self, name: str, value: object, **kwargs: str) -> None:
self.fields.append((name, value, kwargs))
class _FakeContent:
async def iter_any(self):
yield b'data: {"choices":[{"delta":{"content":"hello"}}]}\n\n'
yield b'data: {"usage":{"completion_tokens":1}}\n\n'
yield b"data: [DONE]\n\n"
class _FakeResponse:
def __init__(self) -> None:
self.status = 200
self.reason = "OK"
self.content = _FakeContent()
async def __aenter__(self):
return self
async def __aexit__(self, exc_type, exc, tb):
return False
class _FakeSession:
def __init__(self) -> None:
self.uploaded_bytes: bytes | None = None
self.upload_filename: str | None = None
self.fields: list[tuple[str, object, dict[str, str]]] | None = None
def post(self, *, url: str, data: _FakeFormData, headers: dict[str, str]):
del url, headers
self.fields = list(data.fields)
_, file_obj, file_kwargs = self.fields[0]
file_obj = cast(_ReadableBinary, file_obj)
self.uploaded_bytes = file_obj.read()
self.upload_filename = file_kwargs.get("filename")
return _FakeResponse()
class _FakeProfileSession:
def __init__(self) -> None:
self.url: str | None = None
self.headers: dict[str, str] | None = None
def post(self, *, url: str, headers: dict[str, str]):
self.url = url
self.headers = headers
return _FakeResponse()
def test_asr_dataset_sample_handles_local_audio_paths(tmp_path: Path) -> None:
audio_path = tmp_path / "earnings.wav"
_write_wav(audio_path, duration_s=0.1)
dataset = object.__new__(datasets_module.ASRDataset)
dataset.data = [
{
"audio": {
"path": str(audio_path),
"bytes": None,
},
"text": "quarterly earnings call",
}
]
samples = dataset.sample(
tokenizer=_as_tokenizer(_Tokenizer()),
num_requests=1,
output_len=32,
asr_min_audio_len_sec=0.0,
asr_max_audio_len_sec=1.0,
)
assert len(samples) == 1
assert samples[0].multi_modal_data == {"audio_path": str(audio_path)}
assert (
samples[0].prompt == "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>"
)
@pytest.mark.parametrize("has_filepath", [True, False])
def test_asr_dataset_sample_handles_embedded_audio_bytes(
tmp_path: Path, has_filepath: bool
) -> None:
audio_path = tmp_path / "earnings.wav"
_write_wav(audio_path, duration_s=0.1)
test_path = None
if has_filepath:
test_path = audio_path
dataset = object.__new__(datasets_module.ASRDataset)
dataset.data = [
{
"audio": {
"path": test_path,
"bytes": audio_path.read_bytes(),
},
"text": "quarterly earnings call",
}
]
samples = dataset.sample(
tokenizer=_as_tokenizer(_Tokenizer()),
num_requests=1,
output_len=32,
asr_min_audio_len_sec=0.0,
asr_max_audio_len_sec=1.0,
)
assert len(samples) == 1
assert isinstance(samples[0].multi_modal_data, dict)
audio, sample_rate = samples[0].multi_modal_data["audio"]
assert sample_rate == 16_000
assert isinstance(audio, np.ndarray)
assert audio.size > 0
def test_async_request_openai_audio_handles_local_audio_paths(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
audio_path = tmp_path / "earnings.wav"
_write_wav(audio_path, duration_s=0.25)
monkeypatch.setattr(request_func_module.aiohttp, "FormData", _FakeFormData)
session = _FakeSession()
request_input = RequestFuncInput(
prompt="",
api_url="http://localhost:8000/v1/audio/transcriptions",
prompt_len=1,
output_len=32,
model="openai/whisper-large-v3",
multi_modal_content={"audio_path": str(audio_path)},
)
output = asyncio.run(
request_func_module.async_request_openai_audio(request_input, session)
)
assert session.upload_filename == audio_path.name
assert session.uploaded_bytes == audio_path.read_bytes()
assert output.success is True
assert output.generated_text == "hello"
assert output.output_tokens == 1
assert output.input_audio_duration == pytest.approx(0.25, abs=1e-2)
def test_async_request_openai_audio_handles_decoded_audio_arrays(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(request_func_module.aiohttp, "FormData", _FakeFormData)
session = _FakeSession()
request_input = RequestFuncInput(
prompt="",
api_url="http://localhost:8000/v1/audio/transcriptions",
prompt_len=1,
output_len=32,
model="openai/whisper-large-v3",
multi_modal_content={
"audio": (np.zeros(1_600, dtype=np.float32), 16_000),
},
)
output = asyncio.run(
request_func_module.async_request_openai_audio(request_input, session)
)
assert session.upload_filename == "audio.wav"
assert session.uploaded_bytes is not None
assert output.success is True
assert output.generated_text == "hello"
def test_audio_request_rejects_profile_endpoint() -> None:
request_input = RequestFuncInput(
prompt="",
api_url="http://localhost:8000/start_profile",
prompt_len=1,
output_len=32,
model="openai/whisper-large-v3",
)
with pytest.raises(ValueError, match="OpenAI Audio API"):
asyncio.run(
request_func_module.async_request_openai_audio(
request_input, _FakeProfileSession()
)
)
@pytest.mark.parametrize("endpoint", ["start_profile", "stop_profile"])
def test_audio_benchmark_profile_request_has_no_inference_payload(
endpoint: str,
) -> None:
session = _FakeProfileSession()
output = asyncio.run(
request_func_module.async_request_profile(
f"http://localhost:8000/{endpoint}",
session,
extra_headers={"X-Test-Header": "profile"},
)
)
assert output.success is True
assert session.url == f"http://localhost:8000/{endpoint}"
assert session.headers == {"X-Test-Header": "profile"}
_COHERE_ASR_PROMPT = (
"<|startofcontext|><|startoftranscript|>"
"<|emo:undefined|><|en|><|en|><|pnc|><|noitn|>"
"<|notimestamp|><|nodiarize|>"
)
def _make_asr_dataset(tmp_path: Path) -> datasets_module.ASRDataset:
audio_path = tmp_path / "sample.wav"
_write_wav(audio_path, duration_s=0.1)
dataset = object.__new__(datasets_module.ASRDataset)
dataset.data = [
{
"audio": {"path": str(audio_path), "bytes": None},
"text": "hello world",
}
]
return dataset
def test_asr_dataset_cohere_class_name_gets_decoder_prompt(tmp_path: Path) -> None:
dataset = _make_asr_dataset(tmp_path)
samples = dataset.sample(
tokenizer=_as_tokenizer(CohereAsrTokenizer()),
num_requests=1,
output_len=32,
asr_min_audio_len_sec=0.0,
asr_max_audio_len_sec=1.0,
)
assert len(samples) == 1
assert samples[0].prompt == _COHERE_ASR_PROMPT
def test_asr_dataset_cohere_name_or_path_fallback_gets_decoder_prompt(
tmp_path: Path,
) -> None:
dataset = _make_asr_dataset(tmp_path)
samples = dataset.sample(
tokenizer=_as_tokenizer(_CohereNameOnlyTokenizer()),
num_requests=1,
output_len=32,
asr_min_audio_len_sec=0.0,
asr_max_audio_len_sec=1.0,
)
assert len(samples) == 1
assert samples[0].prompt == _COHERE_ASR_PROMPT
def test_asr_dataset_unknown_tokenizer_gets_empty_prompt(tmp_path: Path) -> None:
dataset = _make_asr_dataset(tmp_path)
samples = dataset.sample(
tokenizer=_as_tokenizer(_Tokenizer(name_or_path="some-other/asr-model")),
num_requests=1,
output_len=32,
asr_min_audio_len_sec=0.0,
asr_max_audio_len_sec=1.0,
)
assert len(samples) == 1
assert samples[0].prompt == ""
-341
View File
@@ -1,341 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import argparse
import json
from pathlib import Path
from unittest.mock import MagicMock, patch
import pytest
from transformers import AutoTokenizer, PreTrainedTokenizerBase
from vllm.benchmarks.datasets import BFCLDataset, get_samples
def _patch_hf_api(side_effect):
"""Return a patch context that swaps `hf_api()` to a stub whose
`.hf_hub_download` attribute uses `side_effect`."""
fake_api = MagicMock()
fake_api.hf_hub_download.side_effect = side_effect
return patch("vllm.benchmarks.datasets.datasets.hf_api", return_value=fake_api)
@pytest.fixture(scope="session")
def hf_tokenizer() -> PreTrainedTokenizerBase:
return AutoTokenizer.from_pretrained("openai-community/gpt2")
_FAKE_ROWS = {
"simple": [
{
"id": "simple_0",
"question": [
[
{
"role": "user",
"content": "What is 2+2?",
}
]
],
"function": [
{
"name": "add",
"description": "Add two numbers.",
"parameters": {
"type": "dict",
"properties": {
"a": {"type": "integer", "description": "first"},
"b": {"type": "float", "description": "second"},
},
"required": ["a", "b"],
},
}
],
},
],
"live_simple": [
{
"id": "live_simple_0",
"question": [[{"role": "user", "content": "Tell me the weather."}]],
"function": [
{
"name": "get_weather",
"description": "Get weather.",
"parameters": {
"type": "dict",
"properties": {
"city": {"type": "any", "description": "city"},
"coords": {"type": "tuple", "description": "coords"},
},
"required": ["city"],
},
}
],
},
],
}
def _write_fake_files(tmp_path: Path) -> dict[str, Path]:
"""Write fake BFCL JSONL files mimicking the HF repo layout."""
paths = {}
for category, rows in _FAKE_ROWS.items():
p = tmp_path / f"BFCL_v3_{category}.json"
with p.open("w") as f:
for row in rows:
f.write(json.dumps(row) + "\n")
paths[category] = p
return paths
def _args_for_bfcl(categories: list[str] | None) -> argparse.Namespace:
return argparse.Namespace(
dataset_name="hf",
dataset_path="gorilla-llm/Berkeley-Function-Calling-Leaderboard",
hf_name=None,
hf_subset=None,
hf_split=None,
hf_output_len=64,
disable_shuffle=True,
num_prompts=2,
no_oversample=False,
no_stream=True,
seed=0,
request_id_prefix="",
trust_remote_code=False,
skip_chat_template=False,
enable_multimodal_chat=False,
backend="openai-chat",
bfcl_categories=categories,
)
@pytest.mark.benchmark
def test_bfcl_dataset_translates_schema_and_attaches_tools(
hf_tokenizer: PreTrainedTokenizerBase, tmp_path: Path
) -> None:
"""BFCLDataset should translate schemas to OpenAI tool format, set
`messages` directly on SampleRequest, and attach tools/tool_choice via
request_overrides."""
paths = _write_fake_files(tmp_path)
def fake_download(_repo, filename, **_kwargs):
category = filename.removeprefix("BFCL_v3_").removesuffix(".json")
return str(paths[category])
args = _args_for_bfcl(categories=["simple", "live_simple"])
with _patch_hf_api(fake_download):
samples = get_samples(args, hf_tokenizer)
assert len(samples) == 2
for s in samples:
assert s.chat_messages is not None
assert isinstance(s.chat_messages, list)
assert s.chat_messages[0]["role"] == "user"
assert s.request_overrides is not None
assert "tools" in s.request_overrides
assert s.request_overrides["tool_choice"] == "auto"
# messages must NOT leak into request_overrides — it has its own
# typed field on SampleRequest.
assert "messages" not in s.request_overrides
tools = s.request_overrides["tools"]
assert len(tools) == 1
tool = tools[0]
assert tool["type"] == "function"
# Translated schema: dict -> object, float -> number,
# any -> string, tuple -> array.
params = tool["function"]["parameters"]
assert params["type"] == "object"
for prop in params["properties"].values():
assert prop["type"] in {"integer", "number", "string", "array"}
@pytest.mark.benchmark
def test_bfcl_dataset_requires_openai_chat_backend(
hf_tokenizer: PreTrainedTokenizerBase,
) -> None:
args = _args_for_bfcl(categories=["simple"])
args.backend = "openai"
with pytest.raises(ValueError, match="openai-chat"):
get_samples(args, hf_tokenizer)
@pytest.mark.benchmark
def test_bfcl_dataset_missing_category_raises_clear_error(
hf_tokenizer: PreTrainedTokenizerBase,
) -> None:
"""A typo'd category should produce an actionable ValueError, not an
opaque huggingface_hub exception."""
from huggingface_hub.errors import EntryNotFoundError
args = _args_for_bfcl(categories=["simpl"]) # typo
def raise_missing(_repo, filename, **_kwargs):
raise EntryNotFoundError(f"404 Not Found: {filename}")
with (
_patch_hf_api(raise_missing),
pytest.raises(ValueError, match=r"BFCL category 'simpl' not found"),
):
get_samples(args, hf_tokenizer)
@pytest.mark.benchmark
def test_chat_backend_uses_messages_field_when_set() -> None:
"""When RequestFuncInput.chat_messages is set, the chat backend must use
it verbatim and skip default content construction from `prompt`."""
import asyncio
from vllm.benchmarks.lib.endpoint_request_func import (
RequestFuncInput,
async_request_openai_chat_completions,
)
captured: dict = {}
class _FakeResp:
status = 500
reason = "stop-after-capture"
content = None
async def __aenter__(self):
return self
async def __aexit__(self, *exc):
return False
class _FakeSession:
def post(self, url, json, headers): # noqa: A002
captured["url"] = url
captured["payload"] = json
return _FakeResp()
messages = [
{"role": "system", "content": "you are helpful"},
{"role": "user", "content": "call add(3, 4)"},
]
req = RequestFuncInput(
prompt="IGNORED",
api_url="http://localhost:0/v1/chat/completions",
prompt_len=10,
output_len=16,
model="test-model",
chat_messages=messages,
extra_body={"tools": [{"type": "function", "function": {"name": "add"}}]},
)
asyncio.run(
async_request_openai_chat_completions(
request_func_input=req, session=_FakeSession()
)
)
payload = captured["payload"]
assert payload["messages"] is messages, (
"chat backend must forward RequestFuncInput.chat_messages verbatim "
"instead of constructing a default user message from `prompt`"
)
# extra_body still merges in as before (shallow, per-request wins).
assert payload["tools"][0]["function"]["name"] == "add"
@pytest.mark.benchmark
def test_bfcl_prompt_len_includes_tools(tmp_path: Path) -> None:
"""prompt_len must reflect tokens from both messages *and* tool schemas,
so percentile buckets and input-distribution summaries aren't biased
low for tool-heavy traffic."""
paths = _write_fake_files(tmp_path)
def fake_download(_repo, filename, **_kwargs):
category = filename.removeprefix("BFCL_v3_").removesuffix(".json")
return str(paths[category])
captured: dict = {}
class _FakeTokenizer:
def apply_chat_template(
self, messages, tools=None, tokenize=False, add_generation_prompt=True
):
captured["tools"] = tools
base = " ".join(m.get("content", "") for m in messages)
tool_text = json.dumps(tools) if tools else ""
return base + " " + tool_text
def __call__(self, text):
# 1 "token" per whitespace-separated word.
return type("Enc", (), {"input_ids": text.split()})()
fake = _FakeTokenizer()
args = _args_for_bfcl(categories=["simple"])
args.num_prompts = 1
with _patch_hf_api(fake_download):
samples = get_samples(args, fake)
assert len(samples) == 1
assert captured["tools"] is not None, (
"apply_chat_template must be called with tools= so the schema "
"contributes to the prompt-length estimate"
)
assert len(captured["tools"]) == 1
assert captured["tools"][0]["function"]["name"] == "add"
# Sanity: prompt_len exceeds a messages-only estimate. The fake row's
# user message is "What is 2+2?" (3 whitespace-separated tokens).
assert samples[0].prompt_len > 3
@pytest.mark.benchmark
def test_bfcl_prompt_len_falls_back_when_tokenizer_rejects_tools(
tmp_path: Path,
) -> None:
"""Older tokenizers don't accept tools=; fallback must still produce a
non-zero prompt_len without crashing."""
paths = _write_fake_files(tmp_path)
def fake_download(_repo, filename, **_kwargs):
category = filename.removeprefix("BFCL_v3_").removesuffix(".json")
return str(paths[category])
class _LegacyTokenizer:
def apply_chat_template(self, messages, **kwargs):
if "tools" in kwargs:
raise TypeError("unexpected keyword argument 'tools'")
return " ".join(m.get("content", "") for m in messages)
def __call__(self, text):
return type("Enc", (), {"input_ids": text.split()})()
args = _args_for_bfcl(categories=["simple"])
args.num_prompts = 1
with _patch_hf_api(fake_download):
samples = get_samples(args, _LegacyTokenizer())
assert len(samples) == 1
assert samples[0].prompt_len > 0
@pytest.mark.benchmark
def test_bfcl_schema_translation_is_recursive() -> None:
"""_translate_schema must recurse into nested properties."""
input_schema = {
"type": "dict",
"properties": {
"nested": {
"type": "dict",
"properties": {
"value": {"type": "float"},
"tags": {"type": "tuple", "items": {"type": "any"}},
},
}
},
}
out = BFCLDataset._translate_schema(input_schema)
assert out["type"] == "object"
assert out["properties"]["nested"]["type"] == "object"
assert out["properties"]["nested"]["properties"]["value"]["type"] == "number"
assert out["properties"]["nested"]["properties"]["tags"]["type"] == "array"
nested_props = out["properties"]["nested"]["properties"]
assert nested_props["tags"]["items"]["type"] == "string"
@@ -1,75 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import argparse
import json
from pathlib import Path
import pytest
from vllm.benchmarks.datasets import get_samples
class _RecordingTokenizer:
"""Minimal tokenizer stub that records the kwargs forwarded to
apply_chat_template, so we can assert chat_template_kwargs propagation
without loading a real model/template."""
def __init__(self) -> None:
self.captured_kwargs: dict | None = None
self.chat_template = "dummy-template"
def apply_chat_template(
self,
conversation,
add_generation_prompt: bool = True,
tokenize: bool = False,
**kwargs,
) -> str:
self.captured_kwargs = kwargs
return conversation[0]["content"]
def __call__(self, text: str):
return argparse.Namespace(input_ids=list(range(len(text.split()))))
def _args(dataset_path: str, chat_template_kwargs) -> argparse.Namespace:
return argparse.Namespace(
dataset_name="custom",
dataset_path=dataset_path,
disable_shuffle=True,
num_prompts=1,
custom_output_len=32,
skip_chat_template=False,
chat_template_kwargs=chat_template_kwargs,
no_oversample=False,
seed=0,
request_id_prefix="",
)
def _write_one(path: Path) -> None:
path.write_text(json.dumps({"prompt": "hello world"}) + "\n")
@pytest.mark.benchmark
def test_chat_template_kwargs_forwarded(tmp_path: Path) -> None:
"""--chat-template-kwargs must reach the client-side apply_chat_template."""
jsonl = tmp_path / "data.jsonl"
_write_one(jsonl)
tok = _RecordingTokenizer()
get_samples(_args(str(jsonl), {"thinking": True}), tok)
assert tok.captured_kwargs == {"thinking": True}
@pytest.mark.benchmark
def test_chat_template_kwargs_default_is_noop(tmp_path: Path) -> None:
"""When not provided, no extra kwargs are passed (existing behavior)."""
jsonl = tmp_path / "data.jsonl"
_write_one(jsonl)
tok = _RecordingTokenizer()
get_samples(_args(str(jsonl), None), tok)
assert tok.captured_kwargs == {}
@@ -1,77 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import argparse
import json
from pathlib import Path
import pytest
from transformers import AutoTokenizer, PreTrainedTokenizerBase
from vllm.benchmarks.datasets import get_samples
@pytest.fixture(scope="session")
def hf_tokenizer() -> PreTrainedTokenizerBase:
return AutoTokenizer.from_pretrained("openai-community/gpt2")
def _write_jsonl(path: Path, n_rows: int) -> None:
with path.open("w") as f:
for i in range(n_rows):
f.write(json.dumps({"prompt": f"row {i}: unique prompt content."}) + "\n")
def _args_for_custom(dataset_path: str, seed: int) -> argparse.Namespace:
return argparse.Namespace(
dataset_name="custom",
dataset_path=dataset_path,
disable_shuffle=False,
num_prompts=30,
custom_output_len=32,
skip_chat_template=True,
no_oversample=False,
seed=seed,
request_id_prefix="",
)
@pytest.mark.benchmark
def test_custom_dataset_seed_propagates(
hf_tokenizer: PreTrainedTokenizerBase, tmp_path: Path
) -> None:
"""--seed must control the CustomDataset shuffle used by get_samples.
Without the fix, CustomDataset was instantiated without random_seed,
so its load-time shuffle always used DEFAULT_SEED=0 regardless of
args.seed, causing every run with --dataset-name custom to pick the
same subset of rows from a larger file.
"""
jsonl = tmp_path / "data.jsonl"
_write_jsonl(jsonl, n_rows=60)
samples_a = get_samples(_args_for_custom(str(jsonl), seed=0), hf_tokenizer)
samples_b = get_samples(_args_for_custom(str(jsonl), seed=42), hf_tokenizer)
prompts_a = {s.prompt for s in samples_a}
prompts_b = {s.prompt for s in samples_b}
assert len(prompts_a) == 30
assert len(prompts_b) == 30
assert prompts_a != prompts_b
@pytest.mark.benchmark
def test_custom_dataset_same_seed_is_deterministic(
hf_tokenizer: PreTrainedTokenizerBase, tmp_path: Path
) -> None:
"""Same --seed must yield the same CustomDataset subset."""
jsonl = tmp_path / "data.jsonl"
_write_jsonl(jsonl, n_rows=60)
samples_a = get_samples(_args_for_custom(str(jsonl), seed=7), hf_tokenizer)
samples_b = get_samples(_args_for_custom(str(jsonl), seed=7), hf_tokenizer)
prompts_a = [s.prompt for s in samples_a]
prompts_b = [s.prompt for s in samples_b]
assert prompts_a == prompts_b
@@ -1,400 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from argparse import Namespace
from io import BytesIO
from pathlib import Path
from typing import Any, cast
import pybase64 as base64
import pytest
from PIL import Image
import vllm.benchmarks.datasets.datasets as datasets_module
from vllm.benchmarks.datasets import CustomImageDataset, get_samples
from vllm.benchmarks.lib.endpoint_request_func import (
RequestFuncInput,
_get_chat_content,
_get_chat_messages,
)
from vllm.tokenizers import TokenizerLike
pytestmark = pytest.mark.skip_global_cleanup
class _TokenizedPrompt:
def __init__(self, prompt: str) -> None:
self.input_ids = prompt.split()
class _Tokenizer:
def __call__(self, prompt: str) -> _TokenizedPrompt:
return _TokenizedPrompt(prompt)
def _tokenizer() -> TokenizerLike:
return cast(TokenizerLike, _Tokenizer())
def _write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None:
with path.open("w") as f:
for row in rows:
f.write(json.dumps(row) + "\n")
def _write_png(path: Path, color: tuple[int, int, int] = (255, 0, 0)) -> None:
Image.new("RGB", (1, 1), color=color).save(path)
def _decode_data_url(data_url: str) -> tuple[str, bytes]:
prefix, image_base64 = data_url.split(",", 1)
return prefix, base64.b64decode(image_base64)
def _assert_png_data_url(data_url: str) -> None:
prefix, image_bytes = _decode_data_url(data_url)
assert prefix == "data:image/png;base64"
with Image.open(BytesIO(image_bytes)) as image:
image.verify()
def _args_for_custom_image(dataset_path: Path) -> Namespace:
return Namespace(
dataset_name="custom_image",
dataset_path=str(dataset_path),
disable_shuffle=True,
seed=0,
num_prompts=2,
custom_output_len=32,
enable_multimodal_chat=False,
custom_ensure_client_side_data=False,
request_id_prefix="req-",
no_oversample=False,
)
@pytest.mark.benchmark
def test_get_samples_custom_image_cli_path_supports_multi_image_and_content(
tmp_path: Path,
) -> None:
image_a = tmp_path / "chart_a.png"
image_b = tmp_path / "chart_b.png"
image_c = tmp_path / "chart_c.png"
jsonl = tmp_path / "images.jsonl"
_write_jsonl(
jsonl,
[
{
"prompt": "Compare the first two charts.",
"image_files": [str(image_a), str(image_b)],
},
{
"content": [
{"type": "text", "text": "Now compare "},
{"type": "image", "image": str(image_c)},
],
},
],
)
samples = get_samples(_args_for_custom_image(jsonl), _tokenizer())
assert len(samples) == 2
assert samples[0].request_id == "req-0"
assert isinstance(samples[0].multi_modal_data, list)
assert [part["image_url"]["url"] for part in samples[0].multi_modal_data] == [
f"file://{image_a}",
f"file://{image_b}",
]
assert samples[1].request_id == "req-1"
assert samples[1].multi_modal_data is None
assert isinstance(samples[1].prompt, list)
assert samples[1].prompt[0] == {"type": "text", "text": "Now compare "}
assert isinstance(samples[1].prompt[1], dict)
assert samples[1].prompt[1]["image_url"]["url"] == f"file://{image_c}"
@pytest.mark.benchmark
def test_custom_image_dataset_uses_all_image_files(tmp_path: Path) -> None:
image_a = tmp_path / "chart_a.png"
image_b = tmp_path / "chart_b.png"
jsonl = tmp_path / "images.jsonl"
_write_jsonl(
jsonl,
[
{
"prompt": "Compare the charts.",
"image_files": [str(image_a), str(image_b)],
}
],
)
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
samples = dataset.sample(
tokenizer=_tokenizer(),
num_requests=1,
output_len=32,
)
assert len(samples) == 1
sample = samples[0]
assert sample.prompt == "Compare the charts."
assert sample.prompt_len == 3
assert isinstance(sample.multi_modal_data, list)
assert [part["image_url"]["url"] for part in sample.multi_modal_data] == [
f"file://{image_a}",
f"file://{image_b}",
]
@pytest.mark.benchmark
def test_custom_image_dataset_preserves_interleaved_content_order(
tmp_path: Path,
) -> None:
image_a = tmp_path / "chart_a.png"
image_b = tmp_path / "chart_b.png"
jsonl = tmp_path / "images.jsonl"
_write_jsonl(
jsonl,
[
{
"content": [
{"type": "text", "text": "Compare "},
{"type": "image", "image": str(image_a)},
{"type": "text", "text": " with "},
{
"type": "image_url",
"image_url": {
"url": str(image_b),
"detail": "low",
},
},
],
}
],
)
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
samples = dataset.sample(
tokenizer=_tokenizer(),
num_requests=1,
output_len=32,
)
assert len(samples) == 1
sample = samples[0]
assert sample.multi_modal_data is None
assert sample.prompt_len == 2
assert isinstance(sample.prompt, list)
parts = []
for part in sample.prompt:
assert isinstance(part, dict)
parts.append(part)
assert [part["type"] for part in parts] == [
"text",
"image_url",
"text",
"image_url",
]
assert parts[1]["image_url"]["url"] == f"file://{image_a}"
assert parts[3]["image_url"] == {
"url": f"file://{image_b}",
"detail": "low",
}
request_input = RequestFuncInput(
prompt=sample.prompt,
api_url="http://localhost:8000/v1/chat/completions",
prompt_len=sample.prompt_len,
output_len=32,
model="test-model",
)
assert _get_chat_content(request_input) == sample.prompt
@pytest.mark.benchmark
def test_custom_image_dataset_wraps_interleaved_content_for_multimodal_chat(
tmp_path: Path,
) -> None:
image = tmp_path / "chart.png"
jsonl = tmp_path / "images.jsonl"
_write_jsonl(
jsonl,
[
{
"content": [
{"type": "text", "text": "Describe "},
{"type": "image", "image": str(image)},
],
}
],
)
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
samples = dataset.sample(
tokenizer=_tokenizer(),
num_requests=1,
output_len=32,
enable_multimodal_chat=True,
)
sample = samples[0]
assert sample.multi_modal_data is None
assert sample.prompt == [
{
"role": "user",
"content": [
{"type": "text", "text": "Describe "},
{"type": "image_url", "image_url": {"url": f"file://{image}"}},
],
}
]
request_input = RequestFuncInput(
prompt=sample.prompt,
api_url="http://localhost:8000/v1/chat/completions",
prompt_len=sample.prompt_len,
output_len=32,
model="test-model",
)
assert _get_chat_messages(request_input) == sample.prompt
@pytest.mark.benchmark
def test_custom_image_dataset_encodes_image_media_when_requested(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
image_a = tmp_path / "chart_a.png"
image_b = tmp_path / "chart b.png"
_write_png(image_a, color=(255, 0, 0))
_write_png(image_b, color=(0, 255, 0))
data_url = "data:image/png;base64,Zm9v"
remote_url = "https://example.com/chart.png"
original_fetch_image = datasets_module.fetch_image
def fake_fetch_image(image_url: str) -> Image.Image:
if image_url == remote_url:
return Image.new("RGB", (1, 1), color=(0, 0, 255))
return original_fetch_image(image_url)
monkeypatch.setattr(datasets_module, "fetch_image", fake_fetch_image)
jsonl = tmp_path / "images.jsonl"
_write_jsonl(
jsonl,
[
{
"prompt": "Compare the charts.",
"image_files": [
str(image_a),
image_b.as_uri(),
remote_url,
data_url,
],
}
],
)
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
samples = dataset.sample(
tokenizer=_tokenizer(),
num_requests=1,
output_len=32,
ensure_client_side_data=True,
)
assert len(samples) == 1
assert isinstance(samples[0].multi_modal_data, list)
image_urls = [part["image_url"]["url"] for part in samples[0].multi_modal_data]
_assert_png_data_url(image_urls[0])
_assert_png_data_url(image_urls[1])
_assert_png_data_url(image_urls[2])
assert image_urls[3] == data_url
@pytest.mark.benchmark
def test_custom_image_dataset_encodes_interleaved_image_media(
tmp_path: Path,
) -> None:
image_a = tmp_path / "chart_a.png"
image_b = tmp_path / "chart_b.png"
_write_png(image_a, color=(255, 0, 0))
_write_png(image_b, color=(0, 255, 0))
jsonl = tmp_path / "images.jsonl"
_write_jsonl(
jsonl,
[
{
"content": [
{"type": "text", "text": "Compare "},
{"type": "image", "image": str(image_a)},
{
"type": "image_url",
"image_url": {
"url": image_b.as_uri(),
"detail": "low",
},
},
],
}
],
)
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
samples = dataset.sample(
tokenizer=_tokenizer(),
num_requests=1,
output_len=32,
ensure_client_side_data=True,
)
sample = samples[0]
assert isinstance(sample.prompt, list)
parts = []
for part in sample.prompt:
assert isinstance(part, dict)
parts.append(part)
_assert_png_data_url(parts[1]["image_url"]["url"])
_assert_png_data_url(parts[2]["image_url"]["url"])
assert parts[2]["image_url"]["detail"] == "low"
@pytest.mark.benchmark
def test_custom_image_dataset_rejects_invalid_image_media(
tmp_path: Path,
) -> None:
invalid_image = tmp_path / "not_an_image.png"
invalid_image.write_text("not an image")
jsonl = tmp_path / "images.jsonl"
_write_jsonl(
jsonl,
[{"prompt": "Describe the image.", "image_files": [str(invalid_image)]}],
)
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
with pytest.raises(ValueError, match="Invalid image URL"):
dataset.sample(
tokenizer=_tokenizer(),
num_requests=1,
output_len=32,
ensure_client_side_data=True,
)
@pytest.mark.benchmark
def test_custom_image_dataset_rejects_invalid_content_part(
tmp_path: Path,
) -> None:
jsonl = tmp_path / "images.jsonl"
_write_jsonl(jsonl, [{"content": [{"type": "audio", "audio": "clip.wav"}]}])
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
with pytest.raises(ValueError, match="type 'text', 'image', or 'image_url'"):
dataset.sample(
tokenizer=_tokenizer(),
num_requests=1,
output_len=32,
)
@@ -1,510 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Timing-accounting invariants for the streaming benchmark request functions.
serve.py derives TPOT as (latency - ttft) / (output_len - 1), which only means
"average inter-token latency" if latency - ttft == sum(itl). These tests pin
that identity, and pin that the trailing choice-less usage chunk does not
extend the measured end of the request.
Driven against a fake session and a scripted clock: no GPU, model, or socket.
"""
import asyncio
import importlib.util
import json
from pathlib import Path
from typing import Any
import aiohttp
import numpy as np
import pytest
import vllm.benchmarks.lib.endpoint_request_func as request_func_module
from vllm.benchmarks.lib.endpoint_request_func import (
RequestFuncInput,
async_request_openai_audio,
async_request_openai_chat_completions,
async_request_openai_completions,
)
pytestmark = pytest.mark.skip_global_cleanup
AUDIO_SAMPLE_RATE = 16_000
class _ScriptedClock:
"""A deterministic stand-in for time.perf_counter.
Each read advances the clock by exactly one tick, so what a code path
observes depends only on how many times it reads the clock. A stray or
misattributed read then shows up as a whole tick of drift rather than as
sub-microsecond jitter, which keeps these assertions free of flakiness.
"""
def __init__(self, tick: float = 1.0, start: float = 1000.0) -> None:
self.tick = tick
self._now = start
def __call__(self) -> float:
value = self._now
self._now += self.tick
return value
class _FakeTime:
"""Namespace substituted for the module-level time import."""
def __init__(self, perf_counter) -> None:
self.perf_counter = perf_counter
def _sse(payload: dict[str, Any]) -> bytes:
return f"data: {json.dumps(payload)}\n\n".encode()
def _token_chunk(index: int, *, chat: bool) -> bytes:
if chat:
choice = {"index": 0, "delta": {"content": f"tok{index} "}}
else:
choice = {"index": 0, "text": f"tok{index} "}
return _sse({"id": "cmpl-test", "object": "x", "choices": [choice]})
def _usage_chunk(n_tokens: int) -> bytes:
return _sse(
{
"id": "cmpl-test",
"object": "x",
"choices": [],
"usage": {
"prompt_tokens": 7,
"completion_tokens": n_tokens,
"total_tokens": 7 + n_tokens,
},
}
)
def _build_stream(
n_tokens: int,
*,
chat: bool,
ttft_delay: float = 0.0,
itl_delay: float = 0.0,
usage_delay: float = 0.0,
) -> list[tuple[float, bytes]]:
"""Build ``(delay_before_chunk, payload)`` pairs for one streamed response.
``vllm bench serve`` always requests ``stream_options.include_usage``, so a
choice-less usage chunk always trails the final token.
"""
chunks = [
(ttft_delay if i == 0 else itl_delay, _token_chunk(i, chat=chat))
for i in range(n_tokens)
]
chunks.append((usage_delay, _usage_chunk(n_tokens)))
chunks.append((0.0, b"data: [DONE]\n\n"))
return chunks
class _FakeContent:
def __init__(self, chunks: list[tuple[float, bytes]]) -> None:
self._chunks = chunks
async def iter_any(self):
for delay, payload in self._chunks:
if delay:
await asyncio.sleep(delay)
yield payload
# The legacy script iterates ``response.content`` directly.
def __aiter__(self):
return self.iter_any()
class _FakeResponse:
def __init__(self, chunks: list[tuple[float, bytes]]) -> None:
self.status = 200
self.reason = "OK"
self.content = _FakeContent(chunks)
async def __aenter__(self) -> "_FakeResponse":
return self
async def __aexit__(self, exc_type, exc, tb) -> bool:
return False
class _FakeSession:
"""Returns a scripted SSE stream for any request, without touching a socket."""
def __init__(self, chunks: list[tuple[float, bytes]]) -> None:
self._chunks = chunks
def post(self, **kwargs: Any) -> _FakeResponse:
del kwargs
return _FakeResponse(self._chunks)
async def __aenter__(self) -> "_FakeSession":
return self
async def __aexit__(self, exc_type, exc, tb) -> bool:
return False
# (id, request function, api url, is_chat_payload)
STREAMING_ENDPOINTS = [
(
"completions",
async_request_openai_completions,
"http://test/v1/completions",
False,
),
(
"chat",
async_request_openai_chat_completions,
"http://test/v1/chat/completions",
True,
),
(
"audio",
async_request_openai_audio,
"http://test/v1/audio/transcriptions",
True,
),
]
def _make_input(api_url: str, n_tokens: int, endpoint_id: str) -> RequestFuncInput:
multi_modal_content = None
if endpoint_id == "audio":
# A short silent clip; only its duration is read by the request func.
multi_modal_content = {
"audio": (
np.zeros(AUDIO_SAMPLE_RATE // 10, dtype=np.float32),
AUDIO_SAMPLE_RATE,
)
}
return RequestFuncInput(
prompt="hello",
api_url=api_url,
prompt_len=7,
output_len=n_tokens,
model="test-model",
multi_modal_content=multi_modal_content,
)
def _run(
request_func,
api_url: str,
endpoint_id: str,
chunks,
n_tokens: int,
*,
expect_success: bool = True,
):
output = asyncio.run(
request_func(
_make_input(api_url, n_tokens, endpoint_id),
_FakeSession(chunks),
)
)
if expect_success:
assert output.success, output.error
return output
@pytest.mark.parametrize(
"endpoint_id,request_func,api_url,chat",
STREAMING_ENDPOINTS,
ids=[e[0] for e in STREAMING_ENDPOINTS],
)
def test_decode_span_identity_is_exact(
monkeypatch: pytest.MonkeyPatch,
endpoint_id: str,
request_func,
api_url: str,
chat: bool,
) -> None:
"""Latency - ttft must equal sum(itl) exactly.
On a scripted clock this can only drift if a code path reads the clock an
extra time or attributes a read to the wrong event.
"""
tick = 1.0
monkeypatch.setattr(
request_func_module, "time", _FakeTime(_ScriptedClock(tick=tick))
)
n_tokens = 5
chunks = _build_stream(n_tokens, chat=chat)
output = _run(request_func, api_url, endpoint_id, chunks, n_tokens)
assert len(output.itl) == n_tokens - 1
assert output.output_tokens == n_tokens
residual = (output.latency - output.ttft) - sum(output.itl)
assert residual == pytest.approx(0.0, abs=tick / 1000), (
f"{endpoint_id}: (latency - ttft) - sum(itl) = {residual!r}; "
f"TPOT is derived from latency - ttft, so it must match the measured "
f"inter-token latencies"
)
@pytest.mark.parametrize(
"endpoint_id,request_func,api_url,chat",
STREAMING_ENDPOINTS,
ids=[e[0] for e in STREAMING_ENDPOINTS],
)
def test_trailing_usage_chunk_does_not_extend_latency(
monkeypatch: pytest.MonkeyPatch,
endpoint_id: str,
request_func,
api_url: str,
chat: bool,
) -> None:
"""E2E latency must end at the last token, not at the usage trailer.
The usage chunk carries no token, so counting it stretches E2EL (and
therefore TPOT) by the delivery time of one extra SSE frame. The [DONE]
sentinel is already excluded on the same grounds.
"""
tick = 1.0
clock = _ScriptedClock(tick=tick)
monkeypatch.setattr(request_func_module, "time", _FakeTime(clock))
n_tokens = 5
chunks = _build_stream(n_tokens, chat=chat)
output = _run(request_func, api_url, endpoint_id, chunks, n_tokens)
# Clock reads: 1 for the start time, then 1 per non-[DONE] chunk. The last
# token is therefore read at start + n_tokens ticks; the usage chunk one
# tick later.
expected_latency = n_tokens * tick
assert output.latency == pytest.approx(expected_latency, abs=tick / 1000), (
f"{endpoint_id}: latency {output.latency} should stop at the final "
f"token ({expected_latency}), not run on to the usage chunk "
f"({expected_latency + tick})"
)
@pytest.mark.parametrize(
"endpoint_id,request_func,api_url,chat",
STREAMING_ENDPOINTS,
ids=[e[0] for e in STREAMING_ENDPOINTS],
)
def test_decode_span_identity_holds_with_real_clock(
endpoint_id: str,
request_func,
api_url: str,
chat: bool,
) -> None:
"""The identity also holds against the real clock, with real gaps.
The residual differences out the same recorded timestamps, so wall clock
jitter cancels and only float rounding remains; a microsecond bound is
ample rather than tight.
"""
n_tokens = 4
chunks = _build_stream(
n_tokens, chat=chat, ttft_delay=0.02, itl_delay=0.01, usage_delay=0.02
)
output = _run(request_func, api_url, endpoint_id, chunks, n_tokens)
residual = (output.latency - output.ttft) - sum(output.itl)
assert abs(residual) < 1e-6, (
f"{endpoint_id}: (latency - ttft) - sum(itl) drifted by {residual:.9f}s"
)
@pytest.mark.parametrize(
"endpoint_id,request_func,api_url,chat",
STREAMING_ENDPOINTS,
ids=[e[0] for e in STREAMING_ENDPOINTS],
)
def test_usage_only_stream_is_not_reported_as_success(
endpoint_id: str,
request_func,
api_url: str,
chat: bool,
) -> None:
"""A stream with no token chunk must fail rather than report 0 ms.
Only token chunks move the end of the request, so a degenerate stream
carrying nothing but a usage trailer leaves latency at zero. Reporting that
as a success would feed a zero-duration request into the benchmark
aggregates.
"""
chunks = [(0.0, _usage_chunk(0)), (0.0, b"data: [DONE]\n\n")]
output = _run(request_func, api_url, endpoint_id, chunks, 0, expect_success=False)
assert not output.success
assert output.itl == []
assert "TTFT" in output.error
# Significant whitespace at every edge, plus a multibyte character so that a
# split can land inside a UTF-8 sequence.
_BOUNDARY_TEXT = " leading 中 trailing "
@pytest.mark.parametrize(
"endpoint_id,request_func,api_url,chat",
STREAMING_ENDPOINTS,
ids=[e[0] for e in STREAMING_ENDPOINTS],
)
def test_streamed_output_is_invariant_to_transport_chunk_boundaries(
endpoint_id: str,
request_func,
api_url: str,
chat: bool,
) -> None:
"""iter_any() yields arbitrary transport fragments, so parsing must
give the same result no matter where the byte stream is split."""
if chat:
choice: dict[str, Any] = {"index": 0, "delta": {"content": _BOUNDARY_TEXT}}
else:
choice = {"index": 0, "text": _BOUNDARY_TEXT}
# Not _sse(): its ensure_ascii default would escape the multibyte char.
event = json.dumps(
{"id": "cmpl-test", "object": "x", "choices": [choice]}, ensure_ascii=False
)
stream = f"data: {event}\n\ndata: [DONE]\n\n".encode()
async def run_all_splits() -> list[tuple[int, bool, str, str]]:
failures = []
for split_at in range(1, len(stream)):
chunks = [(0.0, stream[:split_at]), (0.0, stream[split_at:])]
output = await request_func(
_make_input(api_url, 1, endpoint_id),
_FakeSession(chunks),
)
if not output.success or output.generated_text != _BOUNDARY_TEXT:
error = output.error.splitlines()[-1] if output.error else ""
failures.append(
(split_at, output.success, output.generated_text, error)
)
return failures
assert asyncio.run(run_all_splits()) == []
# ``benchmarks/backend_request_func.py`` is a standalone script rather than a
# package module, so it is loaded by path. It also builds its own ClientSession,
# so the scripted stream is injected by swapping the module's aiohttp reference.
_LEGACY_PATH = Path(__file__).parents[2] / "benchmarks" / "backend_request_func.py"
class _FakeAiohttp:
"""Stands in for the legacy module's aiohttp import."""
def __init__(self, chunks: list[tuple[float, bytes]]) -> None:
self._chunks = chunks
def __getattr__(self, name: str) -> Any:
return getattr(aiohttp, name)
def ClientSession(self, **kwargs: Any) -> _FakeSession: # noqa: N802
del kwargs
return _FakeSession(self._chunks)
def _load_legacy():
spec = importlib.util.spec_from_file_location(
"legacy_backend_request_func", _LEGACY_PATH
)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
LEGACY_ENDPOINTS = [
(
"completions",
"async_request_openai_completions",
"http://test/v1/completions",
False,
),
(
"chat",
"async_request_openai_chat_completions",
"http://test/v1/chat/completions",
True,
),
(
"audio",
"async_request_openai_audio",
"http://test/v1/audio/transcriptions",
True,
),
]
def _run_legacy(
monkeypatch, func_name, api_url, endpoint_id, chunks, n_tokens, clock=None
):
module = _load_legacy()
monkeypatch.setattr(module, "aiohttp", _FakeAiohttp(chunks))
if clock is not None:
monkeypatch.setattr(module, "time", _FakeTime(clock))
request_func = getattr(module, func_name)
return asyncio.run(request_func(_make_input(api_url, n_tokens, endpoint_id)))
@pytest.mark.parametrize(
"endpoint_id,func_name,api_url,chat",
LEGACY_ENDPOINTS,
ids=[e[0] for e in LEGACY_ENDPOINTS],
)
def test_legacy_decode_span_identity_is_exact(
monkeypatch: pytest.MonkeyPatch,
endpoint_id: str,
func_name: str,
api_url: str,
chat: bool,
) -> None:
"""benchmarks/backend_request_func.py must honour the same identity."""
tick = 1.0
n_tokens = 5
chunks = _build_stream(n_tokens, chat=chat)
output = _run_legacy(
monkeypatch,
func_name,
api_url,
endpoint_id,
chunks,
n_tokens,
clock=_ScriptedClock(tick=tick),
)
assert output.success, output.error
assert len(output.itl) == n_tokens - 1
residual = (output.latency - output.ttft) - sum(output.itl)
assert residual == pytest.approx(0.0, abs=tick / 1000), (
f"legacy {endpoint_id}: (latency - ttft) - sum(itl) = {residual!r}"
)
assert output.latency == pytest.approx(n_tokens * tick, abs=tick / 1000)
@pytest.mark.parametrize(
"endpoint_id,func_name,api_url,chat",
LEGACY_ENDPOINTS,
ids=[e[0] for e in LEGACY_ENDPOINTS],
)
def test_legacy_usage_only_stream_is_not_reported_as_success(
monkeypatch: pytest.MonkeyPatch,
endpoint_id: str,
func_name: str,
api_url: str,
chat: bool,
) -> None:
"""A usage-only stream must fail rather than report a zero-duration success."""
chunks = [(0.0, _usage_chunk(0)), (0.0, b"data: [DONE]\n\n")]
output = _run_legacy(monkeypatch, func_name, api_url, endpoint_id, chunks, 0)
assert not output.success
assert output.itl == []
-178
View File
@@ -1,178 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import subprocess
from dataclasses import replace
from pathlib import Path
from types import SimpleNamespace
import numpy as np
import pytest
from vllm.benchmarks.forward.batch import build_batch
from vllm.benchmarks.forward.main import run_case
from vllm.benchmarks.forward.profile import summarize
from vllm.benchmarks.forward.shapes import jitter_kv_lens, parse_case
from vllm.platforms import current_platform
MODEL_NAME = "meta-llama/Llama-3.2-1B-Instruct"
def _group(block_size: int, replay: int = 0, dcp_sharded: bool = False):
return SimpleNamespace(
kv_cache_spec=SimpleNamespace(
block_size=block_size,
prefix_replay_tokens=replay,
dcp_sharded=dcp_sharded,
)
)
def test_build_batch_layout():
"""The runner sees each request kind as the scheduler would schedule it."""
case = parse_case("c=2*1@101,4@64:3,32,8@48c,16@316")
groups = [_group(16), _group(32, replay=128)]
batch = build_batch(case, groups, 1000, 1000, np.random.default_rng(0))
# (scheduled, computed, is_decode) per request: decode, verify, prefill,
# chunked prefill, then a prefix hit replaying the last 128 hit tokens.
assert list(batch.expected.values()) == [
(1, 100, True),
(1, 100, True),
(4, 60, True),
(32, 0, False),
(8, 40, False),
(16 + 128, 300 - 128, False),
]
output = batch.scheduler_output
assert output.total_num_scheduled_tokens == 1 + 1 + 4 + 32 + 8 + 144
assert len(output.scheduled_spec_decode_tokens["bench-2"]) == 3
assert output.scheduled_new_reqs[-1].replay_start == 300 - 128
for req_id, (scheduled, _, _) in batch.expected.items():
assert len(batch.query_token_ids[req_id]) == scheduled
# Groups share one block pool: block IDs must not overlap.
ranges = sorted(r for group in batch.group_block_ranges for r in group)
assert ranges[0][0] == 1
assert all(a[1] == b[0] for a, b in zip(ranges, ranges[1:]))
with pytest.raises(ValueError, match="KV blocks"):
build_batch(case, groups, 10, 1000, np.random.default_rng(0))
# DCP spreads each block's tokens over the DCP ranks, like the scheduler.
groups = [_group(16, dcp_sharded=True), _group(16)]
batch = build_batch(
parse_case("c=1@640"), groups, 1000, 1000, np.random.default_rng(0), 4
)
assert batch.group_block_ranges == [[(1, 11)], [(11, 51)]]
def test_jitter_kv_lens():
"""Prefixes vary, capped at the max length; queries and fresh prefills don't.
A case's draw depends only on the seed and its name, so adding or reordering
other cases doesn't change it."""
case = parse_case("c=1000*1@1001,4@64:3,32")
jittered = jitter_kv_lens(case, 0.5, 1400, 0)
assert jitter_kv_lens(case, 0.5, 1400, 0) == jittered
other = jitter_kv_lens(replace(case, name="d"), 0.5, 1400, 0)
assert other.requests != jittered.requests
prefixes = np.array([r.prefix_len for r in jittered.requests[:1000]])
assert prefixes.min() >= 500 and max(r.kv_len for r in jittered.requests) == 1400
assert prefixes.std() > 200
assert [r.query_len for r in jittered.requests] == [
r.query_len for r in case.requests
]
assert jittered.requests[-1] == case.requests[-1]
assert jitter_kv_lens(case, 0, 2048, 0) == case
with pytest.raises(ValueError, match="KV length"):
jitter_kv_lens(case, 0, 1000, 0)
@pytest.mark.parametrize("capacity", [0, 2])
def test_bench_forward_rejects_oversized_verify(capacity):
"""Reject drafts beyond the runner's buffer before launching CUDA kernels."""
runner = SimpleNamespace(num_speculative_steps=capacity)
case = parse_case("verify=2*4@64:3")
with pytest.raises(
ValueError,
match=rf"Case 'verify' needs 3 draft tokens.*supports {capacity}",
):
run_case(runner, case, SimpleNamespace())
def test_profile_summary():
"""Each replay's leading restore/rendezvous activities are left out, and
kernels are grouped by name without template or function arguments."""
def act(name, ts, dur, cat="kernel"):
return {"cat": cat, "name": name, "ts": ts, "dur": dur}
replay = [
("Memcpy DtoD (Device -> Device)", "gpu_memcpy", 5),
("void ns::gemm<128>(Params)", "kernel", 10),
("void ns::gemm<64>(Params)", "kernel", 20),
("std::enable_if<(1>0), void>::type ns::norm<float>(float*)", "kernel", 4),
]
events = [{"cat": "cpu_op", "name": "x", "ts": 0, "dur": 1}]
for r in range(2):
for i, (name, cat, dur) in enumerate(replay):
events.append(act(name, 1000 * r + 100 * i, dur, cat))
summary = summarize(events, replays=2, skip=1)
assert summary["pre_forward_us"] == 5
assert summary["busy_us"] == 34
assert summary["span_us"] == 204
assert [(k["name"], k["us"], k["calls"]) for k in summary["kernels"]] == [
("ns::gemm", 30, 2),
("ns::norm", 4, 1),
]
with pytest.raises(ValueError, match="replays"):
summarize(events[:-1], replays=2, skip=1)
@pytest.mark.benchmark
# TODO(woosuk): Support ROCm and re-enable this test there.
@pytest.mark.skipif(
current_platform.is_rocm(), reason="Hangs on ROCm; the bench is CUDA-only."
)
def test_bench_forward(tmp_path):
output = tmp_path / "forward.json"
command = [
"vllm",
"bench",
"forward",
"--model",
MODEL_NAME,
"--max-model-len",
"2048",
"--gpu-memory-utilization",
"0.5",
"--case",
"decode=8*1@512",
"--case",
"prefill=256",
"--case",
"mix=128@256,4*1@512",
"--iters",
"3",
"--samples",
"2",
"--output-json",
str(output),
"--profile-dir",
str(tmp_path / "profile"),
]
result = subprocess.run(command, capture_output=True, text=True)
print(result.stdout)
print(result.stderr)
assert result.returncode == 0, f"Benchmark failed: {result.stderr}"
results = {row["case"]: row for row in json.loads(output.read_text())}
assert list(results) == ["decode", "prefill", "mix"]
for row in results.values():
rank0 = row["ranks"][0]
assert len(rank0["batch_order"]) == len(row["requests"])
assert rank0["finite"] and rank0["graph_deterministic"]
assert row["median_us"] > 0
profile = rank0["profile"]
assert profile["top_kernels"] and profile["span_us"] > 0
summary = json.loads(Path(profile["summary"]).read_text())
assert Path(summary["trace"]).exists()
-171
View File
@@ -1,171 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pandas as pd
import pytest
from vllm.benchmarks.sweep.plot import (
PlotEqualTo,
PlotFilterBase,
PlotFilters,
PlotGreaterThan,
PlotGreaterThanOrEqualTo,
PlotLessThan,
PlotLessThanOrEqualTo,
PlotNotEqualTo,
)
class TestPlotFilters:
"""Test PlotFilter functionality including 'inf' edge case."""
def setup_method(self):
"""Create sample DataFrames for testing."""
# DataFrame with numeric values
self.df_numeric = pd.DataFrame(
{
"request_rate": [1.0, 5.0, 10.0, 50.0, 100.0],
"value": [10, 20, 30, 40, 50],
}
)
# DataFrame with float('inf') - note: string "inf" values are coerced
# to float when loading data, so we only test with float('inf')
self.df_inf_float = pd.DataFrame(
{
"request_rate": [1.0, 5.0, 10.0, float("inf"), float("inf")],
"value": [10, 20, 30, 40, 50],
}
)
@pytest.mark.parametrize(
"target,expected_count",
[
("5.0", 1),
("10.0", 1),
("1.0", 1),
],
)
def test_equal_to_numeric(self, target, expected_count):
"""Test PlotEqualTo with numeric values."""
filter_obj = PlotEqualTo("request_rate", target)
result = filter_obj.apply(self.df_numeric)
assert len(result) == expected_count
def test_equal_to_inf_float(self):
"""Test PlotEqualTo with float('inf')."""
filter_obj = PlotEqualTo("request_rate", "inf")
result = filter_obj.apply(self.df_inf_float)
# Should match both float('inf') entries because float('inf') == float('inf')
assert len(result) == 2
@pytest.mark.parametrize(
"target,expected_count",
[
("5.0", 4), # All except 5.0
("1.0", 4), # All except 1.0
],
)
def test_not_equal_to_numeric(self, target, expected_count):
"""Test PlotNotEqualTo with numeric values."""
filter_obj = PlotNotEqualTo("request_rate", target)
result = filter_obj.apply(self.df_numeric)
assert len(result) == expected_count
def test_not_equal_to_inf_float(self):
"""Test PlotNotEqualTo with float('inf')."""
filter_obj = PlotNotEqualTo("request_rate", "inf")
result = filter_obj.apply(self.df_inf_float)
# Should exclude float('inf') entries
assert len(result) == 3
@pytest.mark.parametrize(
"target,expected_count",
[
("10.0", 2), # 1.0, 5.0
("50.0", 3), # 1.0, 5.0, 10.0
("5.0", 1), # 1.0
],
)
def test_less_than(self, target, expected_count):
"""Test PlotLessThan with numeric values."""
filter_obj = PlotLessThan("request_rate", target)
result = filter_obj.apply(self.df_numeric)
assert len(result) == expected_count
@pytest.mark.parametrize(
"target,expected_count",
[
("10.0", 3), # 1.0, 5.0, 10.0
("5.0", 2), # 1.0, 5.0
],
)
def test_less_than_or_equal_to(self, target, expected_count):
"""Test PlotLessThanOrEqualTo with numeric values."""
filter_obj = PlotLessThanOrEqualTo("request_rate", target)
result = filter_obj.apply(self.df_numeric)
assert len(result) == expected_count
@pytest.mark.parametrize(
"target,expected_count",
[
("10.0", 2), # 50.0, 100.0
("5.0", 3), # 10.0, 50.0, 100.0
],
)
def test_greater_than(self, target, expected_count):
"""Test PlotGreaterThan with numeric values."""
filter_obj = PlotGreaterThan("request_rate", target)
result = filter_obj.apply(self.df_numeric)
assert len(result) == expected_count
@pytest.mark.parametrize(
"target,expected_count",
[
("10.0", 3), # 10.0, 50.0, 100.0
("5.0", 4), # 5.0, 10.0, 50.0, 100.0
],
)
def test_greater_than_or_equal_to(self, target, expected_count):
"""Test PlotGreaterThanOrEqualTo with numeric values."""
filter_obj = PlotGreaterThanOrEqualTo("request_rate", target)
result = filter_obj.apply(self.df_numeric)
assert len(result) == expected_count
@pytest.mark.parametrize(
"filter_str,expected_var,expected_target,expected_type",
[
("request_rate==5.0", "request_rate", "5.0", PlotEqualTo),
("request_rate!=10.0", "request_rate", "10.0", PlotNotEqualTo),
("request_rate<50.0", "request_rate", "50.0", PlotLessThan),
("request_rate<=50.0", "request_rate", "50.0", PlotLessThanOrEqualTo),
("request_rate>10.0", "request_rate", "10.0", PlotGreaterThan),
("request_rate>=10.0", "request_rate", "10.0", PlotGreaterThanOrEqualTo),
("request_rate==inf", "request_rate", "inf", PlotEqualTo),
("request_rate!='inf'", "request_rate", "inf", PlotNotEqualTo),
],
)
def test_parse_str(self, filter_str, expected_var, expected_target, expected_type):
"""Test parsing filter strings."""
filter_obj = PlotFilterBase.parse_str(filter_str)
assert isinstance(filter_obj, expected_type)
assert filter_obj.var == expected_var
assert filter_obj.target == expected_target
def test_parse_str_inf_edge_case(self):
"""Test parsing 'inf' string in filter."""
filter_obj = PlotFilterBase.parse_str("request_rate==inf")
assert isinstance(filter_obj, PlotEqualTo)
assert filter_obj.var == "request_rate"
assert filter_obj.target == "inf"
def test_parse_multiple_filters(self):
"""Test parsing multiple filters."""
filters = PlotFilters.parse_str("request_rate>5.0,value<=40")
assert len(filters) == 2
assert isinstance(filters[0], PlotGreaterThan)
assert isinstance(filters[1], PlotLessThanOrEqualTo)
def test_parse_empty_filter(self):
"""Test parsing empty filter string."""
filters = PlotFilters.parse_str("")
assert len(filters) == 0
-486
View File
@@ -1,486 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import random
from typing import Any, NamedTuple, cast
import numpy as np
import pytest
from transformers import AutoTokenizer, PreTrainedTokenizerBase
from vllm.benchmarks.datasets import (
RandomDataset,
RandomMultiModalDataset,
SampleRequest,
)
@pytest.fixture(scope="session")
def hf_tokenizer() -> PreTrainedTokenizerBase:
# Use a small, commonly available tokenizer
return AutoTokenizer.from_pretrained("openai-community/gpt2")
class Params(NamedTuple):
num_requests: int
prefix_len: int
range_ratio: float
input_len: int
output_len: int
@pytest.fixture(scope="session")
def random_dataset_params() -> Params:
return Params(
num_requests=16, prefix_len=7, range_ratio=0.3, input_len=50, output_len=20
)
def _fingerprint_sample(req: SampleRequest) -> tuple[str, int, int]:
"""Project a SampleRequest into a comparable tuple."""
assert isinstance(req.prompt, str)
return (req.prompt, req.prompt_len, req.expected_output_len)
def _collect_samples(
dataset: RandomDataset,
tokenizer: PreTrainedTokenizerBase,
num_requests: int = 16,
prefix_len: int = 7,
range_ratio: float = 0.3,
input_len: int = 50,
output_len: int = 20,
) -> list[tuple[str, int, int]]:
samples = dataset.sample(
tokenizer=tokenizer,
num_requests=num_requests,
prefix_len=prefix_len,
range_ratio=range_ratio,
input_len=input_len,
output_len=output_len,
)
return [_fingerprint_sample(s) for s in samples]
@pytest.mark.benchmark
def test_random_dataset_same_seed(
hf_tokenizer: PreTrainedTokenizerBase, random_dataset_params: Params
) -> None:
"""Same seed should yield identical outputs, even if global RNGs change.
This guards against accidental reliance on Python's random or np.random
in RandomDataset after moving to numpy.default_rng.
"""
p = random_dataset_params
common_seed = 123
dataset_a = RandomDataset(random_seed=common_seed)
dataset_b = RandomDataset(random_seed=common_seed)
a = _collect_samples(
dataset_a,
hf_tokenizer,
num_requests=p.num_requests,
prefix_len=p.prefix_len,
range_ratio=p.range_ratio,
input_len=p.input_len,
output_len=p.output_len,
)
# Perturb global RNG state to ensure isolation
random.seed(999)
_ = [random.random() for _ in range(100)]
np.random.seed(888)
_ = [np.random.random() for _ in range(100)]
b = _collect_samples(
dataset_b,
hf_tokenizer,
num_requests=p.num_requests,
prefix_len=p.prefix_len,
range_ratio=p.range_ratio,
input_len=p.input_len,
output_len=p.output_len,
)
assert a == b
@pytest.mark.benchmark
def test_random_dataset_different_seeds(
hf_tokenizer: PreTrainedTokenizerBase, random_dataset_params: Params
) -> None:
"""Different seeds should change outputs with overwhelming likelihood."""
p = random_dataset_params
seed_a = 0
dataset_a = RandomDataset(random_seed=seed_a)
a = _collect_samples(
dataset_a,
hf_tokenizer,
num_requests=p.num_requests,
prefix_len=p.prefix_len,
range_ratio=p.range_ratio,
input_len=p.input_len,
output_len=p.output_len,
)
seed_b = 999
dataset_b = RandomDataset(random_seed=seed_b)
# Perturb global RNG with same seed as dataset_a to ensure isolation
random.seed(seed_a)
np.random.seed(seed_a)
b = _collect_samples(
dataset_b,
hf_tokenizer,
num_requests=p.num_requests,
prefix_len=p.prefix_len,
range_ratio=p.range_ratio,
input_len=p.input_len,
output_len=p.output_len,
)
assert a != b
# -----------------------------
# RandomMultiModalDataset tests
# -----------------------------
def _mm_fingerprint_sample(
req: SampleRequest,
) -> tuple[str, int, int, int, list[str]]:
"""Create a compact fingerprint for multimodal samples.
Includes:
- prompt string
- prompt_len
- expected_output_len
- count of multimodal items
- per-item type and URL prefix (e.g., 'data:image/jpeg;base64,')
"""
assert isinstance(req.prompt, str)
items = req.multi_modal_data or []
item_prefixes: list[str] = []
for it in items:
if isinstance(it, dict) and it.get("type") == "image_url":
url = it.get("image_url", {}).get("url", "")
# Only keep a short identifying prefix to avoid huge strings
item_prefixes.append(f"image:{url[:22]}")
elif isinstance(it, dict) and it.get("type") == "video_url":
url = it.get("video_url", {}).get("url", "")
item_prefixes.append(f"video:{url[:22]}")
else:
item_prefixes.append("unknown:")
return (
req.prompt,
req.prompt_len,
req.expected_output_len,
len(items),
item_prefixes,
)
def _collect_mm_samples(
dataset: RandomMultiModalDataset,
tokenizer: PreTrainedTokenizerBase,
*,
num_requests: int = 8,
prefix_len: int = 3,
range_ratio: float = 0.0,
input_len: int = 20,
output_len: int = 5,
base_items_per_request: int = 2,
num_mm_items_range_ratio: float = 0.0,
limit_mm_per_prompt: dict[str, int] | None = None,
bucket_config: dict[tuple[int, int, int], float] | None = None,
enable_multimodal_chat: bool = False,
) -> list[SampleRequest]:
if limit_mm_per_prompt is None:
limit_mm_per_prompt = {"image": 5, "video": 0}
if bucket_config is None:
bucket_config = {(32, 32, 1): 0.5, (52, 64, 1): 0.5}
return dataset.sample(
tokenizer=tokenizer,
num_requests=num_requests,
prefix_len=prefix_len,
range_ratio=range_ratio,
input_len=input_len,
output_len=output_len,
base_items_per_request=base_items_per_request,
num_mm_items_range_ratio=num_mm_items_range_ratio,
limit_mm_per_prompt=limit_mm_per_prompt,
bucket_config=bucket_config,
enable_multimodal_chat=enable_multimodal_chat,
)
@pytest.mark.benchmark
def test_random_mm_same_seed(hf_tokenizer: PreTrainedTokenizerBase) -> None:
seed = 42
ds_a = RandomMultiModalDataset(random_seed=seed)
ds_b = RandomMultiModalDataset(random_seed=seed)
a = _collect_mm_samples(ds_a, hf_tokenizer)
b = _collect_mm_samples(ds_b, hf_tokenizer)
fa = [_mm_fingerprint_sample(s) for s in a]
fb = [_mm_fingerprint_sample(s) for s in b]
assert fa == fb
@pytest.mark.benchmark
def test_random_mm_different_seeds(
hf_tokenizer: PreTrainedTokenizerBase,
) -> None:
ds_a = RandomMultiModalDataset(random_seed=0)
ds_b = RandomMultiModalDataset(random_seed=999)
a = _collect_mm_samples(ds_a, hf_tokenizer)
b = _collect_mm_samples(ds_b, hf_tokenizer)
fa = [_mm_fingerprint_sample(s) for s in a]
fb = [_mm_fingerprint_sample(s) for s in b]
assert fa != fb
@pytest.mark.benchmark
def test_random_mm_respects_limits(
hf_tokenizer: PreTrainedTokenizerBase,
) -> None:
ds = RandomMultiModalDataset(random_seed=0)
# Requesting 3 items with a per-prompt limit of 1 should error per current
# design (dataset refuses to silently clamp below the requested baseline).
with pytest.raises(ValueError):
_collect_mm_samples(
ds,
hf_tokenizer,
num_requests=12,
base_items_per_request=3,
num_mm_items_range_ratio=0.0,
limit_mm_per_prompt={"image": 1, "video": 0},
bucket_config={(32, 32, 1): 1.0},
)
@pytest.mark.benchmark
def test_random_mm_zero_prob_entries_are_removed(
hf_tokenizer: PreTrainedTokenizerBase,
) -> None:
ds = RandomMultiModalDataset(random_seed=0)
# Second bucket has zero probability and should be ignored after
# normalization
samples = _collect_mm_samples(
ds,
hf_tokenizer,
num_requests=6,
base_items_per_request=2,
num_mm_items_range_ratio=0.0,
limit_mm_per_prompt={"image": 10, "video": 0},
bucket_config={(32, 32, 1): 1.0, (52, 64, 1): 0.0},
)
for s in samples:
assert isinstance(s.multi_modal_data, list)
typed_mm = cast(list[dict[str, Any]], s.multi_modal_data)
for it in typed_mm:
assert it.get("type") == "image_url"
@pytest.mark.benchmark
def test_random_mm_zero_items(hf_tokenizer: PreTrainedTokenizerBase) -> None:
ds = RandomMultiModalDataset(random_seed=0)
samples = _collect_mm_samples(
ds,
hf_tokenizer,
num_requests=5,
base_items_per_request=0,
num_mm_items_range_ratio=0.0,
limit_mm_per_prompt={"image": 5, "video": 0},
bucket_config={(32, 32, 1): 1.0},
)
for s in samples:
assert s.multi_modal_data == []
@pytest.mark.benchmark
def test_random_mm_num_items_per_prompt(hf_tokenizer: PreTrainedTokenizerBase) -> None:
ds = RandomMultiModalDataset(random_seed=0)
# Fixed number of images per prompt
# set num_mm_items_range_ratio to 0.0
# TODO: modify video values when video sampling is implemented
samples_fixed_items = _collect_mm_samples(
ds,
hf_tokenizer,
num_requests=5,
base_items_per_request=3,
num_mm_items_range_ratio=0.0,
limit_mm_per_prompt={"image": 3, "video": 0},
bucket_config={(32, 32, 1): 1.0},
)
# Must have 5 requests each with 3 mm items per prompt
assert len(samples_fixed_items) == 5
for s in samples_fixed_items:
mm_data = cast(list[dict[str, Any]], s.multi_modal_data)
assert len(mm_data) == 3
for it in mm_data:
assert it.get("type") == "image_url"
@pytest.mark.benchmark
def test_random_mm_bucket_config_not_mutated(
hf_tokenizer: PreTrainedTokenizerBase,
) -> None:
ds = RandomMultiModalDataset(random_seed=0)
# This bucket config is not normalized to sum to 1
# and has more buckets than requested images
original = {(32, 32, 1): 0.2, (52, 64, 1): 6, (25, 64, 1): 3}
# Keep a snapshot to compare after sampling
snapshot = dict(original)
_ = _collect_mm_samples(
ds,
hf_tokenizer,
num_requests=4,
base_items_per_request=1,
num_mm_items_range_ratio=0.0,
limit_mm_per_prompt={"image": 1, "video": 0},
bucket_config=original,
)
# Ensure the original dict content is unchanged
assert original == snapshot
# Vary number of mm items per prompt
# set num_mm_items_range_ratio to 0.5
samples_varying_items = _collect_mm_samples(
ds,
hf_tokenizer,
num_requests=5,
base_items_per_request=2,
num_mm_items_range_ratio=0.5,
limit_mm_per_prompt={"image": 4, "video": 0},
bucket_config={(32, 32, 1): 1.0},
)
# Must have 5 requests each with less than 4 mm items per prompt
# but at least 1 mm item per prompt
assert len(samples_varying_items) == 5
for s in samples_varying_items:
mm_data = cast(list[dict[str, Any]], s.multi_modal_data)
assert len(mm_data) <= 4
assert len(mm_data) >= 1
for it in mm_data:
assert it.get("type") == "image_url"
@pytest.mark.benchmark
def test_random_mm_video_sampling(hf_tokenizer: PreTrainedTokenizerBase) -> None:
"""Test video sampling functionality in RandomMultiModalDataset."""
ds = RandomMultiModalDataset(random_seed=42)
# Test with video bucket configuration
bucket_config = {
(64, 64, 1): 0.3, # Images
(64, 64, 8): 0.7, # Videos
}
limit_mm_per_prompt = {"image": 2, "video": 2}
samples = _collect_mm_samples(
ds,
hf_tokenizer,
num_requests=5,
base_items_per_request=1,
num_mm_items_range_ratio=0.0,
limit_mm_per_prompt=limit_mm_per_prompt,
bucket_config=bucket_config,
)
assert len(samples) == 5
# Check that we have both images and videos
video_count = 0
image_count = 0
for s in samples:
mm_data = cast(list[dict[str, Any]], s.multi_modal_data)
assert len(mm_data) == 1
item = mm_data[0]
if item.get("type") == "video_url":
video_count += 1
# Verify video URL format
url = item.get("video_url", {}).get("url", "")
assert url.startswith("data:video/mp4;base64,")
elif item.get("type") == "image_url":
image_count += 1
# Verify image URL format
url = item.get("image_url", {}).get("url", "")
assert url.startswith("data:image/jpeg;base64,")
# Should have some videos due to 0.7 probability
assert video_count > 0
assert image_count > 0
@pytest.mark.benchmark
def test_random_mm_video_only_sampling(hf_tokenizer: PreTrainedTokenizerBase) -> None:
"""Test sampling with only video buckets."""
ds = RandomMultiModalDataset(random_seed=42)
bucket_config = {
(64, 64, 8): 1.0, # Only videos
}
limit_mm_per_prompt = {"image": 0, "video": 1}
samples = _collect_mm_samples(
ds,
hf_tokenizer,
num_requests=3,
base_items_per_request=1,
num_mm_items_range_ratio=0.0,
limit_mm_per_prompt=limit_mm_per_prompt,
bucket_config=bucket_config,
)
assert len(samples) == 3
for s in samples:
mm_data = cast(list[dict[str, Any]], s.multi_modal_data)
assert len(mm_data) == 1
item = mm_data[0]
assert item.get("type") == "video_url"
url = item.get("video_url", {}).get("url", "")
assert url.startswith("data:video/mp4;base64,")
@pytest.mark.benchmark
def test_random_mm_video_deterministic_sampling(
hf_tokenizer: PreTrainedTokenizerBase,
) -> None:
"""Test that video sampling is deterministic with same seed."""
seed = 123
ds_a = RandomMultiModalDataset(random_seed=seed)
ds_b = RandomMultiModalDataset(random_seed=seed)
bucket_config = {
(64, 64, 8): 1.0, # Only videos
}
limit_mm_per_prompt = {"image": 0, "video": 1}
a = _collect_mm_samples(
ds_a,
hf_tokenizer,
num_requests=3,
base_items_per_request=1,
num_mm_items_range_ratio=0.0,
limit_mm_per_prompt=limit_mm_per_prompt,
bucket_config=bucket_config,
)
b = _collect_mm_samples(
ds_b,
hf_tokenizer,
num_requests=3,
base_items_per_request=1,
num_mm_items_range_ratio=0.0,
limit_mm_per_prompt=limit_mm_per_prompt,
bucket_config=bucket_config,
)
fa = [_mm_fingerprint_sample(s) for s in a]
fb = [_mm_fingerprint_sample(s) for s in b]
assert fa == fb
@@ -1,434 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
import tempfile
from pathlib import Path
from tempfile import NamedTemporaryFile
from typing import Any, cast
from unittest.mock import Mock
import cv2
import pybase64 as base64
import pytest
from transformers import AutoTokenizer, PreTrainedTokenizerBase
from vllm.benchmarks.datasets import RandomMultiModalDataset, SampleRequest
@pytest.fixture(scope="session")
def hf_tokenizer() -> PreTrainedTokenizerBase:
"""Use a small, commonly available tokenizer."""
return AutoTokenizer.from_pretrained("openai-community/gpt2")
@pytest.fixture
def video_dataset() -> RandomMultiModalDataset:
"""Create a RandomMultiModalDataset instance for testing."""
return RandomMultiModalDataset(random_seed=42)
@pytest.mark.benchmark
def test_generate_synthetic_video_different_seeds():
"""Test that different seeds produce different videos."""
dataset1 = RandomMultiModalDataset(random_seed=123)
dataset2 = RandomMultiModalDataset(random_seed=456)
width, height, num_frames = 64, 48, 8
video1 = dataset1.generate_synthetic_video(width, height, num_frames)
video2 = dataset2.generate_synthetic_video(width, height, num_frames)
# Videos should be different due to different seeds
assert video1["bytes"] != video2["bytes"]
@pytest.mark.benchmark
def test_map_config_to_modality(video_dataset: RandomMultiModalDataset):
"""Test modality mapping for different configurations."""
# Test image configuration (num_frames = 1)
assert video_dataset.map_config_to_modality((256, 256, 1)) == "image"
assert video_dataset.map_config_to_modality((720, 1280, 1)) == "image"
# Test video configurations (num_frames > 1)
assert video_dataset.map_config_to_modality((256, 256, 8)) == "video"
assert video_dataset.map_config_to_modality((720, 1280, 16)) == "video"
assert video_dataset.map_config_to_modality((64, 64, 32)) == "video"
# Test invalid configurations
with pytest.raises(ValueError, match="Invalid multimodal item configuration"):
video_dataset.map_config_to_modality((256, 256, 0))
with pytest.raises(ValueError, match="Invalid multimodal item configuration"):
video_dataset.map_config_to_modality((256, 256, -1))
@pytest.mark.benchmark
def test_generate_mm_item_video(
video_dataset: RandomMultiModalDataset,
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
):
"""Generate a decodable video without leaving temporary files behind."""
monkeypatch.setattr(tempfile, "tempdir", str(tmp_path))
# Test video item generation
video_config = (64, 48, 8) # height, width, num_frames
result = video_dataset.generate_mm_item(video_config)
assert not list(tmp_path.iterdir())
# Check the result structure matches OpenAI API format
assert isinstance(result, dict)
assert result["type"] == "video_url"
assert "video_url" in result
assert "url" in result["video_url"]
# Check that the URL is a data URL with base64 encoded video
url = result["video_url"]["url"]
assert url.startswith("data:video/mp4;base64,")
# Decode and verify the video content
base64_data = url.split(",")[1]
video_bytes = base64.b64decode(base64_data)
assert len(video_bytes) > 0
# Verify the video can be decoded
with NamedTemporaryFile(suffix=".mp4", delete=False) as temp_file:
temp_path = temp_file.name
temp_file.write(video_bytes)
try:
cap = cv2.VideoCapture(temp_path)
assert cap.isOpened()
frame_count = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
frame_width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
frame_height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
assert frame_count == 8
assert frame_width == 48
assert frame_height == 64
cap.release()
finally:
if os.path.exists(temp_path):
os.unlink(temp_path)
@pytest.mark.benchmark
@pytest.mark.parametrize("writer_opened", [False, True], ids=["open", "write"])
def test_generate_synthetic_video_cleans_up_on_error(
video_dataset: RandomMultiModalDataset,
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
writer_opened: bool,
):
"""Release the writer and remove temporary storage when encoding fails."""
monkeypatch.setattr(tempfile, "tempdir", str(tmp_path))
writer = Mock(spec=cv2.VideoWriter)
writer.isOpened.return_value = writer_opened
writer.write.side_effect = RuntimeError("Failed to write video frame")
monkeypatch.setattr(cv2, "VideoWriter", lambda *args, **kwargs: writer)
error = (
"Failed to write video frame"
if writer_opened
else "Failed to create video writer"
)
with pytest.raises(RuntimeError, match=error):
video_dataset.generate_synthetic_video(48, 64, 8)
assert not list(tmp_path.iterdir())
writer.release.assert_called_once_with()
@pytest.mark.benchmark
def test_generate_mm_item_image(video_dataset: RandomMultiModalDataset):
"""Test generating multimodal items for image configurations."""
# Test image item generation
image_config = (64, 48, 1) # height, width, num_frames=1
result = video_dataset.generate_mm_item(image_config)
# Check the result structure matches OpenAI API format
assert isinstance(result, dict)
assert result["type"] == "image_url"
assert "image_url" in result
assert "url" in result["image_url"]
# Check that the URL is a data URL with base64 encoded image
url = result["image_url"]["url"]
assert url.startswith("data:image/jpeg;base64,")
@pytest.mark.benchmark
def test_generate_mm_item_invalid_config(video_dataset: RandomMultiModalDataset):
"""Test error handling for invalid configurations."""
with pytest.raises(ValueError, match="Invalid multimodal item configuration"):
video_dataset.generate_mm_item((256, 256, 0))
@pytest.mark.benchmark
def test_sample_with_video_buckets(
video_dataset: RandomMultiModalDataset, hf_tokenizer: PreTrainedTokenizerBase
):
"""Test sampling with video bucket configurations."""
# Configure bucket with video probability > 0
bucket_config = {
(64, 64, 1): 0.3, # Images
(64, 64, 8): 0.7, # Videos
}
limit_mm_per_prompt = {"image": 5, "video": 3}
samples = video_dataset.sample(
tokenizer=hf_tokenizer,
num_requests=5,
base_items_per_request=2,
num_mm_items_range_ratio=0.0,
limit_mm_per_prompt=limit_mm_per_prompt,
bucket_config=bucket_config,
input_len=20,
output_len=5,
)
assert len(samples) == 5
# Check that samples contain both images and videos
video_count = 0
image_count = 0
for sample in samples:
assert isinstance(sample, SampleRequest)
assert sample.multi_modal_data is not None
assert isinstance(sample.multi_modal_data, list)
mm_data = cast(list[dict[str, Any]], sample.multi_modal_data)
assert len(mm_data) == 2 # base_items_per_request
for item in mm_data:
if item["type"] == "video_url":
video_count += 1
# Verify video URL format
url = item["video_url"]["url"]
assert url.startswith("data:video/mp4;base64,")
elif item["type"] == "image_url":
image_count += 1
# Verify image URL format
url = item["image_url"]["url"]
assert url.startswith("data:image/jpeg;base64,")
# Should have some videos due to 0.7 probability
assert video_count > 0
assert image_count > 0
@pytest.mark.benchmark
def test_sample_video_only_buckets(
video_dataset: RandomMultiModalDataset, hf_tokenizer: PreTrainedTokenizerBase
):
"""Test sampling with only video buckets."""
bucket_config = {
(64, 64, 8): 1.0, # Only videos
}
limit_mm_per_prompt = {"image": 0, "video": 2}
samples = video_dataset.sample(
tokenizer=hf_tokenizer,
num_requests=3,
base_items_per_request=1,
num_mm_items_range_ratio=0.0,
limit_mm_per_prompt=limit_mm_per_prompt,
bucket_config=bucket_config,
input_len=20,
output_len=5,
)
assert len(samples) == 3
for sample in samples:
assert isinstance(sample, SampleRequest)
assert sample.multi_modal_data is not None
assert isinstance(sample.multi_modal_data, list)
mm_data = cast(list[dict[str, Any]], sample.multi_modal_data)
assert len(mm_data) == 1
item = mm_data[0]
assert item["type"] == "video_url"
url = item["video_url"]["url"]
assert url.startswith("data:video/mp4;base64,")
@pytest.mark.benchmark
def test_sample_respects_video_limits(
video_dataset: RandomMultiModalDataset, hf_tokenizer: PreTrainedTokenizerBase
):
"""Test that sampling respects video limits per prompt."""
bucket_config = {
(64, 64, 8): 1.0, # Only videos
}
# Set very low video limit
limit_mm_per_prompt = {"image": 0, "video": 1}
samples = video_dataset.sample(
tokenizer=hf_tokenizer,
num_requests=3,
base_items_per_request=1,
num_mm_items_range_ratio=0.0,
limit_mm_per_prompt=limit_mm_per_prompt,
bucket_config=bucket_config,
input_len=20,
output_len=5,
)
assert len(samples) == 3
for sample in samples:
mm_data = cast(list[dict[str, Any]], sample.multi_modal_data)
assert len(mm_data) <= 1 # Should respect video limit
@pytest.mark.benchmark
def test_sample_mixed_buckets_with_zero_probability(
video_dataset: RandomMultiModalDataset, hf_tokenizer: PreTrainedTokenizerBase
):
"""Test sampling with mixed buckets including zero probability entries."""
bucket_config = {
(64, 64, 1): 0.5, # Images
(64, 64, 8): 0.5, # Videos
(128, 128, 16): 0.0, # Zero probability videos (should be ignored)
}
limit_mm_per_prompt = {"image": 2, "video": 2}
samples = video_dataset.sample(
tokenizer=hf_tokenizer,
num_requests=4,
base_items_per_request=2,
num_mm_items_range_ratio=0.0,
limit_mm_per_prompt=limit_mm_per_prompt,
bucket_config=bucket_config,
input_len=20,
output_len=5,
)
assert len(samples) == 4
# Should only see 64x64 videos, not 128x128 videos
for sample in samples:
mm_data = cast(list[dict[str, Any]], sample.multi_modal_data)
for item in mm_data:
if item["type"] == "video_url":
# Decode video to verify dimensions
url = item["video_url"]["url"]
base64_data = url.split(",")[1]
video_bytes = base64.b64decode(base64_data)
with NamedTemporaryFile(suffix=".mp4", delete=False) as temp_file: # noqa
temp_path = temp_file.name
temp_file.write(video_bytes)
try:
cap = cv2.VideoCapture(temp_path)
frame_width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
frame_height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
cap.release()
# Should be 64x64, not 128x128
assert frame_width == 64
assert frame_height == 64
finally:
if os.path.exists(temp_path):
os.unlink(temp_path)
@pytest.mark.benchmark
def test_sample_deterministic_with_videos(hf_tokenizer: PreTrainedTokenizerBase):
"""Test that sampling with videos is deterministic with same seed."""
dataset1 = RandomMultiModalDataset(random_seed=123)
dataset2 = RandomMultiModalDataset(random_seed=123)
bucket_config = {
(64, 64, 1): 0.3, # Images
(64, 64, 8): 0.7, # Videos
}
limit_mm_per_prompt = {"image": 2, "video": 2}
samples1 = dataset1.sample(
tokenizer=hf_tokenizer,
num_requests=3,
base_items_per_request=1,
num_mm_items_range_ratio=0.0,
limit_mm_per_prompt=limit_mm_per_prompt,
bucket_config=bucket_config,
input_len=20,
output_len=5,
)
samples2 = dataset2.sample(
tokenizer=hf_tokenizer,
num_requests=3,
base_items_per_request=1,
num_mm_items_range_ratio=0.0,
limit_mm_per_prompt=limit_mm_per_prompt,
bucket_config=bucket_config,
input_len=20,
output_len=5,
)
assert len(samples1) == len(samples2)
# Compare multimodal data
for s1, s2 in zip(samples1, samples2):
assert s1.multi_modal_data == s2.multi_modal_data
@pytest.mark.benchmark
def test_sample_different_seeds_produce_different_videos(
hf_tokenizer: PreTrainedTokenizerBase,
):
"""Test that different seeds produce different video content."""
dataset1 = RandomMultiModalDataset(random_seed=123)
dataset2 = RandomMultiModalDataset(random_seed=456)
bucket_config = {
(64, 64, 8): 1.0, # Only videos
}
limit_mm_per_prompt = {"image": 0, "video": 1}
samples1 = dataset1.sample(
tokenizer=hf_tokenizer,
num_requests=2,
base_items_per_request=1,
num_mm_items_range_ratio=0.0,
limit_mm_per_prompt=limit_mm_per_prompt,
bucket_config=bucket_config,
input_len=20,
output_len=5,
)
samples2 = dataset2.sample(
tokenizer=hf_tokenizer,
num_requests=2,
base_items_per_request=1,
num_mm_items_range_ratio=0.0,
limit_mm_per_prompt=limit_mm_per_prompt,
bucket_config=bucket_config,
input_len=20,
output_len=5,
)
# Video content should be different
for s1, s2 in zip(samples1, samples2):
mm_data1 = cast(list[dict[str, Any]], s1.multi_modal_data)
mm_data2 = cast(list[dict[str, Any]], s2.multi_modal_data)
assert len(mm_data1) == len(mm_data2) == 1
url1 = mm_data1[0]["video_url"]["url"]
url2 = mm_data2[0]["video_url"]["url"]
assert url1 != url2 # Different video content
@@ -1,453 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the ``openai-responses`` backend of ``vllm bench serve``.
The Responses stream differs from Chat Completions in ways the benchmark client
has to get right: every SSE message is labelled with an ``event:`` line,
reasoning and output text arrive as separate token-bearing events, and a
request ends with ``response.completed`` rather than with its last token. These
tests drive the real request function against a local SSE server so the
resulting latency and usage numbers are covered end to end.
"""
import asyncio
import json
from collections.abc import Sequence
import aiohttp
import pytest
from aiohttp import web
from aiohttp.test_utils import TestServer
from vllm.benchmarks.lib.endpoint_request_func import (
ASYNC_REQUEST_FUNCS,
OPENAI_COMPATIBLE_BACKENDS,
RequestFuncInput,
RequestFuncOutput,
async_request_openai_responses,
)
# These tests never touch torch, so skip the GPU cleanup fixture.
pytestmark = pytest.mark.skip_global_cleanup
MODEL = "openai/gpt-oss-20b"
# Gaps between scripted events. GAP_TOKEN is much larger than GAP_PREFILL so
# that a time to first token measured at the wrong event is unambiguous.
GAP_PREFILL = 0.15
GAP_TOKEN = 0.5
GAP_FINALIZE = 0.3
def sse(event_type: str, payload: dict) -> bytes:
"""Frame one event the way ``/v1/responses`` does, with an ``event:`` line.
``ensure_ascii=False`` matches the server, whose pydantic
``model_dump_json`` emits non-ASCII characters unescaped.
"""
data = json.dumps({"type": event_type, **payload}, ensure_ascii=False)
return f"event: {event_type}\ndata: {data}\n\n".encode()
def usage_event(
input_tokens: int,
output_tokens: int,
event_type: str = "response.completed",
status: str = "completed",
):
return sse(
event_type,
{
"response": {
"status": status,
"usage": {
"input_tokens": input_tokens,
"output_tokens": output_tokens,
"total_tokens": input_tokens + output_tokens,
},
}
},
)
def text_delta(delta: str) -> bytes:
return sse("response.output_text.delta", {"delta": delta})
def reasoning_delta(delta: str) -> bytes:
return sse("response.reasoning_text.delta", {"delta": delta})
# The metadata events the server sends before any token. None of these may
# start the TTFT clock or contribute an ITL sample.
PREAMBLE = (
sse("response.created", {"response": {"status": "in_progress"}})
+ sse("response.in_progress", {"response": {"status": "in_progress"}})
+ sse("response.output_item.added", {"output_index": 0})
+ sse("response.reasoning_part.added", {"output_index": 0})
+ sse("response.content_part.added", {"output_index": 0})
)
# The `.done` family the server sends after the last delta. These repeat the
# accumulated text, so timing them would add bogus ITL samples and counting
# their payload would duplicate the output.
DONE_EVENTS = (
sse("response.reasoning_text.done", {"text": "The user asks "})
+ sse("response.reasoning_part.done", {"output_index": 0})
+ sse("response.output_text.done", {"text": "Rayleigh scattering."})
+ sse("response.content_part.done", {"output_index": 0})
+ sse("response.output_item.done", {"output_index": 0})
)
async def _serve(
script: Sequence[bytes | float],
*,
status: int = 200,
chunk_size: int | None = None,
seen: dict | None = None,
**request_kwargs,
) -> RequestFuncOutput:
"""Run the request function against a server replaying ``script``.
Args:
script: Byte strings to write, interleaved with floats to sleep for.
status: HTTP status to respond with. Non-200 skips the script.
chunk_size: If set, split every write into chunks of this many bytes to
emulate fragmented transport.
seen: If given, populated with the request payload.
request_kwargs: Extra fields for the ``RequestFuncInput``.
"""
async def handler(request: web.Request) -> web.StreamResponse:
if seen is not None:
seen["payload"] = await request.json()
seen["headers"] = dict(request.headers)
if status != 200:
return web.Response(status=status, reason="Bad Request")
response = web.StreamResponse(headers={"Content-Type": "text/event-stream"})
await response.prepare(request)
for item in script:
if isinstance(item, float):
await asyncio.sleep(item)
elif chunk_size is None:
await response.write(item)
else:
for i in range(0, len(item), chunk_size):
await response.write(item[i : i + chunk_size])
await response.write_eof()
return response
app = web.Application()
app.router.add_post("/v1/responses", handler)
server = TestServer(app)
await server.start_server()
try:
request_input = RequestFuncInput(
prompt="Why is the sky blue?",
api_url=str(server.make_url("/v1/responses")),
prompt_len=5,
output_len=16,
model=MODEL,
**request_kwargs,
)
async with aiohttp.ClientSession() as session:
return await async_request_openai_responses(request_input, session)
finally:
await server.close()
def run(script: Sequence[bytes | float], **kwargs) -> RequestFuncOutput:
return asyncio.run(_serve(script, **kwargs))
def test_backend_is_registered_as_openai_compatible():
assert ASYNC_REQUEST_FUNCS["openai-responses"] is async_request_openai_responses
# Membership enables --ignore-eos on random datasets and the sampling
# parameter flags, all of which ResponsesRequest accepts.
assert "openai-responses" in OPENAI_COMPATIBLE_BACKENDS
def test_request_payload_is_accepted_by_the_server_schema():
"""Every field sent must be a real ResponsesRequest field.
This is what makes the backend safe to list in OPENAI_COMPATIBLE_BACKENDS:
the flags that membership enables have to reach the model, and the server
drops unknown fields without complaining.
"""
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
sampling = {
"temperature": 0.7,
"top_p": 0.9,
"top_k": 20,
"min_p": 0.05,
"frequency_penalty": 0.2,
"presence_penalty": 0.1,
"repetition_penalty": 1.05,
}
seen: dict = {}
output = run(
[PREAMBLE, text_delta("hi"), usage_event(5, 1)],
seen=seen,
extra_body=sampling,
ignore_eos=True,
model_name="served-alias",
request_id="bench-7",
extra_headers={"x-team": "perf"},
)
assert output.success
# --served-model-name wins over --model, as on the other backends.
assert seen["payload"]["model"] == "served-alias"
# The route is behind validate_json_request, which 415s a missing type.
assert seen["headers"]["Content-Type"] == "application/json"
assert seen["headers"]["x-request-id"] == "bench-7"
assert seen["headers"]["x-team"] == "perf"
assert seen["payload"] == {
"model": "served-alias",
"input": "Why is the sky blue?",
"max_output_tokens": 16,
"stream": True,
"store": False,
"ignore_eos": True,
**sampling,
}
assert set(seen["payload"]) <= set(ResponsesRequest.model_fields)
ResponsesRequest.model_validate(seen["payload"])
def test_latency_accounting_matches_the_other_endpoints():
"""`latency - ttft` must equal `sum(itl)`.
`serve.py` derives TPOT as `(latency - ttft) / (output_len - 1)`, which only
means average inter-token latency when that identity holds. #55508 fixed the
same accounting on the chat and audio backends; the trailing events after
the last token must not extend the request.
"""
output = run(
[
PREAMBLE,
GAP_PREFILL / 2,
# A keep-alive comment sent during prefill is not a token.
b": keep-alive\n\n",
GAP_PREFILL / 2,
reasoning_delta("The user asks "),
GAP_TOKEN,
text_delta("Rayleigh"),
text_delta(" scattering."),
DONE_EVENTS,
GAP_FINALIZE,
usage_event(5, 8),
]
)
assert output.success
# Reasoning is timed but not returned, matching how `openai-chat` treats
# `DeltaMessage.reasoning`.
assert output.generated_text == "Rayleigh scattering."
# Neither the response.created/in_progress/output_item.added preamble nor
# the keep-alive comment may start the clock, and the reasoning delta must,
# so TTFT lands on the prefill gap rather than near zero or out at the
# first output text delta.
assert GAP_PREFILL * 0.8 <= output.ttft < GAP_PREFILL + GAP_TOKEN * 0.5
# One ITL per token event after the first: the gap before "Rayleigh" and
# the back-to-back delta that follows it.
assert len(output.itl) == 2
assert output.itl[0] >= GAP_TOKEN * 0.5
assert output.itl[1] < GAP_TOKEN * 0.5
# The identity is exact, so no wall-clock margin is involved. The .done
# family and the GAP_FINALIZE wait before response.completed sit outside
# the request, exactly as the trailing usage chunk does on chat.
assert output.latency - output.ttft == pytest.approx(sum(output.itl), abs=1e-9)
assert output.start_time > 0.0
def test_usage_is_taken_from_the_terminal_event():
output = run([PREAMBLE, text_delta("blue"), usage_event(17, 4)])
assert output.success
assert output.prompt_len == 17
assert output.output_tokens == 4
@pytest.mark.parametrize("event_type", ["response.completed", "response.incomplete"])
def test_incomplete_response_is_a_successful_request(event_type: str):
"""Hitting max_output_tokens is the Responses form of finish_reason=length.
vLLM reports it as `response.completed` carrying `status: "incomplete"`.
The spec's dedicated `response.incomplete` event is accepted as well.
"""
output = run(
[
PREAMBLE,
text_delta("truncated"),
usage_event(17, 16, event_type=event_type, status="incomplete"),
]
)
assert output.success
assert output.generated_text == "truncated"
assert output.output_tokens == 16
@pytest.mark.parametrize("chunk_size", [1, 3, 7, 64])
def test_output_survives_transport_chunk_boundaries(chunk_size: int):
"""Splits inside the event line, the JSON, or a multibyte character."""
text = " leading 中 trailing "
output = run(
[PREAMBLE, text_delta(text), usage_event(5, 6)],
chunk_size=chunk_size,
)
assert output.success
assert output.generated_text == text
assert output.output_tokens == 6
@pytest.mark.parametrize(
"char", ["\u2028", "\u2029", "\u0085"], ids=["u2028", "u2029", "u0085"]
)
def test_unicode_line_separators_do_not_truncate_the_payload(char: str):
"""The server emits these unescaped and they are legal inside a JSON string.
`str.splitlines()` treats them as line breaks, which would cut the `data:`
line short and fail the request on valid model output.
"""
text = f"before{char}after"
output = run([PREAMBLE, text_delta(text), usage_event(5, 3)])
assert output.success, output.error
assert output.generated_text == text
def test_empty_data_line_is_skipped():
"""A bare `data:` line carries no JSON and must not fail the request."""
output = run([PREAMBLE, b"data:\n\n", text_delta("ok"), usage_event(5, 1)])
assert output.success, output.error
assert output.generated_text == "ok"
def test_prompt_len_falls_back_when_usage_is_absent():
"""A terminal event without usage keeps the locally computed prompt length."""
output = run(
[
PREAMBLE,
text_delta("hi"),
sse("response.completed", {"response": {"status": "completed"}}),
]
)
assert output.success, output.error
assert output.prompt_len == 5
def test_multi_line_data_payload_is_joined_with_newlines():
"""The SSE spec allows one payload to span several `data:` lines."""
script: list[bytes | float] = [
PREAMBLE,
b"event: response.output_text.delta\n"
b'data: {"type": "response.output_text.delta",\n'
b'data: "delta": "ok"}\n\n',
usage_event(5, 1),
]
output = run(script)
assert output.success, output.error
assert output.generated_text == "ok"
def test_done_sentinel_is_skipped():
"""Other Responses servers send `[DONE]`, but vLLM does not."""
output = run([PREAMBLE, text_delta("ok"), usage_event(5, 1), b"data: [DONE]\n\n"])
assert output.success, output.error
assert output.generated_text == "ok"
def test_error_before_any_token_reports_the_server_error():
"""The server's message must win over the generic no-TTFT message."""
output = run(
[PREAMBLE, b'event: error\ndata: {"type": "error", "message": "boom"}\n\n']
)
assert not output.success
assert "boom" in output.error
def test_http_error_is_not_reported_as_success():
output = run([], status=400)
assert not output.success
assert output.error
def test_stream_without_tokens_is_not_reported_as_success():
"""A stream that finishes without generating anything has no TTFT."""
output = run([PREAMBLE, usage_event(5, 0)])
assert not output.success
assert "TTFT" in output.error
def test_truncated_stream_is_not_reported_as_success():
"""A stream cut short after a token has no usage and an understated E2EL."""
output = run([PREAMBLE, text_delta("partial")])
assert not output.success
assert "terminal event" in output.error
def test_mid_stream_error_is_not_reported_as_success():
"""An error payload mid-stream must not be counted as a success.
vLLM cannot emit this today: its streaming error path fails to validate
against the event union, so a generation failure truncates the stream
instead. Proxies and other Responses servers do send it.
"""
output = run(
[
PREAMBLE,
text_delta("partial"),
b'event: error\ndata: {"error": {"message": "boom", "code": 500}}\n\n',
]
)
assert not output.success
assert "boom" in output.error
def test_data_only_framing_is_still_parsed():
"""Some proxies drop the ``event:`` line and forward only ``data:``."""
script: list[bytes | float] = [
b'data: {"type": "response.output_text.delta", "delta": "ok"}\n\n',
usage_event(5, 1),
]
output = run(script)
assert output.success
assert output.generated_text == "ok"
def test_endpoint_path_is_validated():
request_input = RequestFuncInput(
prompt="hi",
api_url="http://localhost:8000/v1/chat/completions",
prompt_len=1,
output_len=1,
model=MODEL,
)
async def call() -> None:
async with aiohttp.ClientSession() as session:
await async_request_openai_responses(request_input, session)
with pytest.raises(ValueError, match="responses"):
asyncio.run(call())
@@ -1,61 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Keep the `vllm bench serve` flag snapshot in sync for the Rust CLI parity test.
`rust/src/bench/tests/cli_parity.rs` asserts that every flag in the snapshot is
either a known `vllm-bench` clap flag/alias or explicitly allowlisted as
Python-only, so `VLLM_USE_RUST_BENCH=1` delegation cannot silently reject
documented flags. This test keeps the snapshot itself current.
Run this file directly to regenerate the snapshot:
.venv/bin/python tests/benchmarks/test_rust_bench_cli_parity.py
"""
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[2]
SNAPSHOT = REPO_ROOT / "rust" / "src" / "bench" / "tests" / "python_serve_flags.txt"
HEADER = """\
# Long CLI flags of the Python `vllm bench serve` parser
# (vllm.benchmarks.serve.add_cli_args), sorted; --help excluded.
# Regenerate: .venv/bin/python tests/benchmarks/test_rust_bench_cli_parity.py
# Consumed by rust/src/bench/tests/cli_parity.rs, which requires every flag
# below to be a known vllm-bench flag/alias or a PYTHON_ONLY allowlist entry.
"""
def _current_flags() -> list[str]:
from vllm.benchmarks.serve import add_cli_args
from vllm.utils.argparse_utils import FlexibleArgumentParser
parser = FlexibleArgumentParser()
add_cli_args(parser)
return sorted(
{
opt
for action in parser._actions
for opt in action.option_strings
if opt.startswith("--") and opt != "--help"
}
)
def _snapshot_flags() -> list[str]:
lines = SNAPSHOT.read_text().splitlines()
return [line.strip() for line in lines if line.strip() and not line.startswith("#")]
def test_serve_flag_snapshot_is_current():
assert SNAPSHOT.is_file(), f"missing snapshot {SNAPSHOT}"
assert _current_flags() == _snapshot_flags(), (
"`vllm bench serve` flags changed. Regenerate the snapshot with "
"`.venv/bin/python tests/benchmarks/test_rust_bench_cli_parity.py`, "
"then for any added flag either support it in rust/src/bench/src/cli.rs "
"or add it to PYTHON_ONLY in rust/src/bench/tests/cli_parity.rs"
)
if __name__ == "__main__":
SNAPSHOT.write_text(HEADER + "\n".join(_current_flags()) + "\n")
print(f"wrote {SNAPSHOT}")
-258
View File
@@ -1,258 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import numpy as np
import pytest
from vllm.benchmarks.datasets.utils import get_sampling_params
from vllm.tokenizers import TokenizerLike
class _FakeTokenizer(TokenizerLike):
"""Minimal tokenizer implementing the TokenizerLike protocol
for testing get_sampling_params."""
def __init__(self, vocab_size: int = 1000, num_special_tokens: int = 0) -> None:
self._vocab_size = vocab_size
self._num_special_tokens = num_special_tokens
# -- Properties required by TokenizerLike --
@classmethod
def from_pretrained(cls, path_or_repo_id, *a, **kw):
return cls()
@property
def vocab_size(self) -> int:
return self._vocab_size
@property
def all_special_tokens(self) -> list[str]:
return []
@property
def all_special_ids(self) -> list[int]:
return []
@property
def bos_token_id(self) -> int:
return 0
@property
def eos_token_id(self) -> int:
return 1
@property
def pad_token_id(self) -> int:
return 2
@property
def is_fast(self) -> bool:
return False
@property
def max_token_id(self) -> int:
return self._vocab_size - 1
@property
def max_chars_per_token(self) -> int:
return 4
@property
def truncation_side(self) -> str:
return "right"
def num_special_tokens_to_add(self) -> int:
return self._num_special_tokens
def __call__(self, text, text_pair=None, **kw): # type: ignore[override]
raise NotImplementedError
def get_vocab(self) -> dict[str, int]:
return {}
def get_added_vocab(self) -> dict[str, int]:
return {}
def encode(self, text, **kw) -> list[int]: # type: ignore[override]
raise NotImplementedError
def apply_chat_template(self, messages, **kw): # type: ignore[override]
raise NotImplementedError
def convert_tokens_to_ids(self, tokens):
raise NotImplementedError
def convert_tokens_to_string(self, tokens: list[str]) -> str:
raise NotImplementedError
def decode(self, ids, skip_special_tokens: bool = False) -> str:
raise NotImplementedError
def convert_ids_to_tokens(
self, ids, skip_special_tokens: bool = False
) -> list[str]:
raise NotImplementedError
class TestGetSamplingParams:
"""Tests for ``get_sampling_params`` in ``vllm.benchmarks.datasets.shared``."""
# -- helpers --
@staticmethod
def _tok(vocab_size: int = 1000, num_special: int = 0) -> _FakeTokenizer:
return _FakeTokenizer(vocab_size=vocab_size, num_special_tokens=num_special)
# -- return shape / dtype --
def test_returns_three_arrays(self):
rng = np.random.default_rng(0)
result = get_sampling_params(rng, 5, 0.0, 100, 50, self._tok())
assert len(result) == 3
for arr in result:
assert isinstance(arr, np.ndarray)
@pytest.mark.parametrize("n", [1, 10, 100])
def test_output_length_matches_num_requests(self, n: int):
rng = np.random.default_rng(42)
input_lens, output_lens, offsets = get_sampling_params(
rng, n, 0.0, 64, 32, self._tok()
)
assert input_lens.shape == (n,)
assert output_lens.shape == (n,)
assert offsets.shape == (n,)
# -- fixed lengths (range_ratio = 0) --
def test_zero_range_ratio_gives_constant_lengths(self):
rng = np.random.default_rng(7)
input_lens, output_lens, _ = get_sampling_params(
rng, 20, 0.0, 128, 64, self._tok()
)
assert np.all(input_lens == 128)
assert np.all(output_lens == 64)
def test_special_tokens_subtracted_from_input_only(self):
rng = np.random.default_rng(7)
input_lens, output_lens, _ = get_sampling_params(
rng, 10, 0.0, 100, 50, self._tok(num_special=4)
)
# real_input_len = 100 - 4 = 96, range_ratio 0 → all 96
assert np.all(input_lens == 96)
# special tokens are not subtracted from output length
assert np.all(output_lens == 50)
# -- range ratios --
def test_input_range_bounds(self):
rng = np.random.default_rng(0)
ratio = 0.5
base = 200
input_lens, _, _ = get_sampling_params(
rng, 500, {"input": ratio, "output": 0.0}, base, 50, self._tok()
)
lo = int(np.floor(base * (1 - ratio)))
hi = int(np.ceil(base * (1 + ratio)))
assert np.all(input_lens >= lo)
assert np.all(input_lens <= hi)
def test_output_range_bounds(self):
rng = np.random.default_rng(0)
ratio = 0.3
base = 100
_, output_lens, _ = get_sampling_params(
rng, 500, {"input": 0.0, "output": ratio}, 50, base, self._tok()
)
lo = max(1, int(np.floor(base * (1 - ratio))))
hi = int(np.ceil(base * (1 + ratio)))
assert np.all(output_lens >= lo)
assert np.all(output_lens <= hi)
def test_output_low_clamped_to_one(self):
"""Even with a high ratio that would push output_low to 0,
the function clamps it to 1."""
rng = np.random.default_rng(0)
# output_len=1, ratio=0.99 → floor(1*0.01)=0, should clamp to 1
_, output_lens, _ = get_sampling_params(
rng, 50, {"input": 0.0, "output": 0.99}, 100, 1, self._tok()
)
assert np.all(output_lens >= 1)
# -- offsets bounded by vocab_size --
@pytest.mark.parametrize("vocab", [100, 32000, 128256])
def test_offsets_within_vocab(self, vocab: int):
rng = np.random.default_rng(0)
_, _, offsets = get_sampling_params(
rng, 200, 0.0, 64, 32, self._tok(vocab_size=vocab)
)
assert np.all(offsets >= 0)
assert np.all(offsets < vocab)
# -- reproducibility --
def test_same_seed_same_results(self):
tok = self._tok()
rr = {"input": 0.3, "output": 0.2}
a = get_sampling_params(np.random.default_rng(42), 50, rr, 256, 64, tok)
b = get_sampling_params(np.random.default_rng(42), 50, rr, 256, 64, tok)
for arr_a, arr_b in zip(a, b):
np.testing.assert_array_equal(arr_a, arr_b)
def test_different_seed_different_results(self):
tok = self._tok()
rr = {"input": 0.3, "output": 0.2}
a = get_sampling_params(np.random.default_rng(0), 50, rr, 256, 64, tok)
b = get_sampling_params(np.random.default_rng(1), 50, rr, 256, 64, tok)
# Extremely unlikely all three arrays match with different seeds
assert not all(np.array_equal(arr_a, arr_b) for arr_a, arr_b in zip(a, b))
# -- validation / error paths --
@pytest.mark.parametrize("bad_ratio", [-0.1, 1.0, 1.5])
def test_invalid_input_range_ratio(self, bad_ratio: float):
rng = np.random.default_rng(0)
with pytest.raises(ValueError, match="input_range_ratio"):
get_sampling_params(
rng, 10, {"input": bad_ratio, "output": 0.0}, 100, 50, self._tok()
)
@pytest.mark.parametrize("bad_ratio", [-0.1, 1.0, 1.5])
def test_invalid_output_range_ratio(self, bad_ratio: float):
rng = np.random.default_rng(0)
with pytest.raises(ValueError, match="output_range_ratio"):
get_sampling_params(
rng, 10, {"input": 0.0, "output": bad_ratio}, 100, 50, self._tok()
)
def test_invalid_dict_missing_keys(self):
rng = np.random.default_rng(0)
with pytest.raises(ValueError, match="input.*output"):
get_sampling_params(rng, 10, {"input": 0.1}, 100, 50, self._tok())
def test_input_len_zero_with_special_tokens(self):
"""input_len < num_special_tokens → real_input_len = 0, which is fine
(range [0, 0])."""
rng = np.random.default_rng(0)
input_lens, _, _ = get_sampling_params(
rng, 5, 0.0, 5, 50, self._tok(num_special=10)
)
# real_input_len = max(0, 5 - 10) = 0
assert np.all(input_lens == 0)
# -- edge cases --
def test_single_request(self):
rng = np.random.default_rng(0)
i, o, off = get_sampling_params(rng, 1, 0.0, 100, 50, self._tok())
assert i.shape == (1,)
assert o.shape == (1,)
assert off.shape == (1,)
def test_large_num_requests(self):
rng = np.random.default_rng(0)
i, o, off = get_sampling_params(rng, 10_000, 0.5, 512, 128, self._tok())
assert i.shape == (10_000,)
assert o.shape == (10_000,)
assert off.shape == (10_000,)
@@ -1,163 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Regression test for --skip-tokenizer-init with --dataset-name custom.
Before the fix (introduced by #39896), running:
vllm bench serve \
--backend vllm-pooling \
--dataset-name custom \
--dataset-path <path> \
--model ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11 \
--endpoint /pooling \
--skip-tokenizer-init \
...
raised immediately with:
AssertionError: Tokenizer must be initialized before loading dataset
even though CustomDataset.sample() already handles tokenizer=None.
This test exercises main_async() directly so it catches any regression
re-introduced at the serve.py level, not just inside get_samples().
"""
import argparse
import asyncio
import json
from pathlib import Path
from unittest.mock import AsyncMock, patch
import pytest
import vllm.benchmarks.serve as serve_module
# Exact prompt payload from the failing benchmark run against a
# Prithvi-EO-2.0 pooling endpoint (URL-in / base64-out format).
_PRITHVI_PROMPT = {
"data": {
"data": "https://huggingface.co/christian-pinto/Prithvi-EO-2.0-300M-TL-VLLM/resolve/main/India_900498_S2Hand.tif",
"data_format": "url",
"out_data_format": "b64_json",
"indices": [1, 2, 3, 8, 11, 12],
},
"priority": 0,
"softmax": False,
}
def _write_dataset(path: Path) -> None:
path.write_text(json.dumps({"prompt": _PRITHVI_PROMPT}) + "\n")
def _args(dataset_path: str) -> argparse.Namespace:
"""Reproduce the argparse.Namespace that serve.py builds from the
failing command, including skip_tokenizer_init=True."""
return argparse.Namespace(
# dataset
dataset_name="custom",
dataset_path=dataset_path,
disable_shuffle=False,
num_prompts=1,
custom_output_len=256,
skip_chat_template=True,
chat_template_kwargs=None,
no_oversample=False,
seed=0,
request_id_prefix="bench-",
# model / tokenizer
model="ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11",
served_model_name=None,
tokenizer=None,
tokenizer_mode="auto",
trust_remote_code=False,
skip_tokenizer_init=True, # <-- the flag under test
# backend / endpoint
backend="vllm-pooling",
base_url="http://127.0.0.1:8000",
host="127.0.0.1",
port=8000,
endpoint="/pooling",
header=None,
insecure=False,
# traffic
request_rate=16.0,
burstiness=1.0,
max_concurrency=None,
probe_request_rate=0.0,
# misc serve args that main_async reads before reaching get_samples
plot_timeline=False,
plot_dataset_stats=False,
self_timed=None,
metadata=None,
label=None,
logprobs=None,
use_beam_search=False,
ignore_eos=False,
goodput=None,
percentile_metrics="ttft,tpot,itl,e2el",
metric_percentiles="25,50,75,99",
save_result=False,
append_result=False,
result_dir=".",
result_filename=None,
num_warmups=0,
profile=False,
disable_tqdm=True,
lora_modules=None,
lora_assignment="random",
ramp_up_strategy=None,
ramp_up_start_rps=None,
ramp_up_end_rps=None,
ready_check_timeout_sec=0,
extra_body=None,
top_p=None,
top_k=None,
min_p=None,
temperature=None,
frequency_penalty=None,
presence_penalty=None,
repetition_penalty=None,
save_detailed=False,
input_len=None,
output_len=None,
)
@pytest.mark.benchmark
def test_main_async_skip_tokenizer_init_does_not_raise(tmp_path: Path) -> None:
"""main_async must not raise AssertionError when skip_tokenizer_init=True
and dataset_name='custom'.
On main (before the fix) this test fails with:
AssertionError: Tokenizer must be initialized before loading dataset
"""
dataset_path = tmp_path / "dataset_url_input_india.jsonl"
_write_dataset(dataset_path)
args = _args(str(dataset_path))
# Patch benchmark() so we never make real HTTP requests — the regression
# triggers before benchmark() is ever called, so this just keeps the test
# fast and hermetic.
mock_result = {
"completed": 1,
"failed": 0,
"total_input_tokens": 1,
"total_output_tokens": 1,
"request_throughput": 1.0,
"output_throughput": 1.0,
"total_token_throughput": 1.0,
"input_lens": [],
"output_lens": [],
"ttfts": [],
"itls": [],
"generated_texts": [],
"errors": [],
"duration": 1.0,
}
with patch.object(
serve_module, "benchmark", new=AsyncMock(return_value=mock_result)
):
# Must NOT raise AssertionError
asyncio.run(serve_module.main_async(args))
-369
View File
@@ -1,369 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for ``vllm bench throughput`` CLI.
Throughput reuses ``bench serve``'s ``datasets.get_samples`` dispatch; the
LoRA-assignment and MMVU cases below cover the pieces that are throughput-only
(serve has no analogue), keeping the two benchmarks aligned without duplicating
serve-side dataset coverage.
"""
import subprocess
from argparse import Namespace
from types import SimpleNamespace
import pytest
from transformers import AutoTokenizer, PreTrainedTokenizerBase
from vllm.benchmarks.datasets import SampleRequest
from vllm.benchmarks.throughput import (
_run_vllm_chat_requests,
_to_serve_args,
add_cli_args,
assign_loras,
get_requests,
)
from vllm.utils.argparse_utils import FlexibleArgumentParser
MODEL_NAME = "meta-llama/Llama-3.2-1B-Instruct"
@pytest.mark.benchmark
def test_bench_throughput():
command = [
"vllm",
"bench",
"throughput",
"--model",
MODEL_NAME,
"--input-len",
"32",
"--output-len",
"1",
"--enforce-eager",
"--load-format",
"dummy",
]
result = subprocess.run(command, capture_output=True, text=True)
print(result.stdout)
print(result.stderr)
assert result.returncode == 0, f"Benchmark failed: {result.stderr}"
def test_bench_throughput_accepts_custom_audio_args():
parser = FlexibleArgumentParser()
add_cli_args(parser)
args = parser.parse_args(
[
"--dataset-name",
"custom_audio",
"--dataset-path",
"audio.jsonl",
"--no-oversample",
"--custom-output-len",
"32",
"--enable-multimodal-chat",
]
)
assert args.dataset_name == "custom_audio"
assert args.no_oversample
assert args.custom_output_len == 32
assert args.enable_multimodal_chat
def test_vllm_chat_requests_include_multimodal_content():
class FakeLLM:
def __init__(self):
self.prompts = None
def chat(self, prompts, sampling_params, use_tqdm):
del sampling_params, use_tqdm
self.prompts = prompts
return []
llm = FakeLLM()
audio_content = {
"type": "input_audio",
"input_audio": {"data": "abc", "format": "wav"},
}
request = SampleRequest(
prompt="Transcribe this audio.",
prompt_len=1,
expected_output_len=8,
multi_modal_data=audio_content,
)
_run_vllm_chat_requests(
llm,
[request],
n=1,
disable_detokenize=False,
do_profile=False,
prequeue_requests=False,
)
assert llm.prompts == [
[
{
"role": "user",
"content": [
{"type": "text", "text": "Transcribe this audio."},
audio_content,
],
}
]
]
@pytest.fixture(scope="session")
def hf_tokenizer() -> PreTrainedTokenizerBase:
# Small, commonly available tokenizer.
return AutoTokenizer.from_pretrained("openai-community/gpt2")
# -----------------------------
# LoRA assignment (throughput-only post-processing)
# -----------------------------
def _sr(i: int) -> SampleRequest:
return SampleRequest(prompt=f"prompt {i}", prompt_len=4, expected_output_len=2)
def _lora_args(
*,
lora_path: str | None,
max_loras: int,
lora_assignment: str = "random",
) -> SimpleNamespace:
return SimpleNamespace(
lora_path=lora_path,
max_loras=max_loras,
lora_assignment=lora_assignment,
)
@pytest.mark.benchmark
def test_assign_loras_noop_without_lora_path() -> None:
"""No lora_path -> requests returned unchanged, no LoRA attached."""
reqs = [_sr(i) for i in range(5)]
out = assign_loras(reqs, _lora_args(lora_path=None, max_loras=4))
assert out is reqs
assert all(r.lora_request is None for r in out)
@pytest.mark.benchmark
def test_assign_loras_round_robin() -> None:
"""Round-robin assigns deterministic index % max_loras + 1 IDs."""
reqs = [_sr(i) for i in range(6)]
out = assign_loras(
reqs,
_lora_args(lora_path="/tmp/lora", max_loras=3, lora_assignment="round-robin"),
)
assert [r.lora_request.lora_int_id for r in out] == [1, 2, 3, 1, 2, 3]
@pytest.mark.benchmark
def test_assign_loras_random_in_range() -> None:
"""Random assignment stays in [1, max_loras] and covers all IDs."""
reqs = [_sr(i) for i in range(200)]
out = assign_loras(
reqs,
_lora_args(lora_path="/tmp/lora", max_loras=5, lora_assignment="random"),
)
ids = [r.lora_request.lora_int_id for r in out]
assert all(1 <= i <= 5 for i in ids)
assert len(set(ids)) == 5
# -----------------------------
# Issue #50838: MMVU via throughput
# -----------------------------
@pytest.mark.benchmark
def test_get_requests_resolves_mmvu(
monkeypatch: pytest.MonkeyPatch,
hf_tokenizer: PreTrainedTokenizerBase,
) -> None:
"""Issue #50838: --dataset-path yale-nlp/MMVU is accepted by throughput.
Previously throughput's validate_args rejected MMVU ("is not supported by
hf dataset"). It now resolves through the shared get_samples. The HF
download is stubbed so the resolution + dispatch path is exercised without
network access.
"""
import vllm.benchmarks.datasets.datasets as dsmod
class _StubMMVUDataset:
# MMVU is multimodal in content but, like the real class, does not set
# IS_MULTIMODAL -- so it is not subject to the backend gate.
IS_MULTIMODAL = False
SUPPORTED_DATASET_PATHS = {"yale-nlp/MMVU": lambda x: x["question"]}
def __init__(self, **kwargs):
self.__dict__.update(kwargs)
def sample(self, *, num_requests, tokenizer, **kwargs): # noqa: ARG002
return [
SampleRequest(prompt="q", prompt_len=1, expected_output_len=1)
for _ in range(num_requests)
]
monkeypatch.setattr(dsmod, "MMVUDataset", _StubMMVUDataset)
parser = FlexibleArgumentParser()
add_cli_args(parser)
args = parser.parse_args(
[
"--dataset-name",
"hf",
"--dataset-path",
"yale-nlp/MMVU",
"--backend",
"vllm-chat",
"--num-prompts",
"3",
]
)
requests = get_requests(args, hf_tokenizer)
assert len(requests) == 3
assert all(isinstance(r, SampleRequest) for r in requests)
@pytest.mark.benchmark
def test_get_requests_allows_multimodal_on_plain_vllm_backend(
monkeypatch: pytest.MonkeyPatch,
hf_tokenizer: PreTrainedTokenizerBase,
) -> None:
"""Regression: --backend vllm must accept multimodal datasets such as ASR.
_run_vllm_requests forwards multi_modal_data straight to LLM.generate, so
the vllm-chat-only gate in get_samples rejected a backend that always
worked.
"""
import vllm.benchmarks.datasets.datasets as dsmod
class _StubASRDataset:
IS_MULTIMODAL = True
SUPPORTED_DATASET_PATHS = {"openslr/librispeech_asr"}
def __init__(self, **kwargs):
self.__dict__.update(kwargs)
def sample(self, *, num_requests, tokenizer, **kwargs): # noqa: ARG002
return [
SampleRequest(prompt="q", prompt_len=1, expected_output_len=1)
for _ in range(num_requests)
]
monkeypatch.setattr(dsmod, "ASRDataset", _StubASRDataset)
parser = FlexibleArgumentParser()
add_cli_args(parser)
args = parser.parse_args(
[
"--dataset-name",
"hf",
"--dataset-path",
"openslr/librispeech_asr",
"--backend",
"vllm",
"--num-prompts",
"3",
]
)
requests = get_requests(args, hf_tokenizer)
assert len(requests) == 3
assert all(isinstance(r, SampleRequest) for r in requests)
def test_to_serve_args_supports_mm_processor_flags() -> None:
"""The adapter must serve callers without throughput's legacy flags.
``bench mm-processor`` exposes neither --input-len/--output-len/--prefix-len
nor --backend, and leaves --random-prefix-len at its default of 0. The
adapter must not read the legacy flags unconditionally, and must keep the
explicit 0 instead of coercing it to None (which later crashed
RandomMultiModalDataset.get_prefix on ``None <= 0``).
"""
args = Namespace(
random_input_len=32,
random_output_len=1,
random_prefix_len=0,
)
serve_args = _to_serve_args(args)
assert serve_args.random_input_len == 32
assert serve_args.random_output_len == 1
assert serve_args.random_prefix_len == 0
assert serve_args.backend == "vllm-chat"
assert serve_args.enable_multimodal_chat
assert serve_args.no_oversample is False
def test_to_serve_args_keeps_legacy_flag_fallback() -> None:
"""Callers that only set the legacy flags still get them mapped."""
args = Namespace(
input_len=256,
output_len=8,
prefix_len=16,
backend="vllm",
)
serve_args = _to_serve_args(args)
assert serve_args.random_input_len == 256
assert serve_args.random_output_len == 8
assert serve_args.random_prefix_len == 16
assert serve_args.backend == "vllm"
assert not serve_args.enable_multimodal_chat
@pytest.mark.benchmark
def test_get_requests_random_mm_with_mm_processor_args(
hf_tokenizer: PreTrainedTokenizerBase,
) -> None:
"""Regression: bench mm-processor's random-mm sampling must not crash.
get_requests feeds mm-processor's parsed args (no legacy length/backend
flags) through the shared get_samples dispatch. With the default
--random-prefix-len 0 this previously crashed inside
RandomMultiModalDataset.get_prefix with a TypeError.
"""
from vllm.benchmarks.mm_processor import add_cli_args as add_mm_cli_args
parser = FlexibleArgumentParser()
add_mm_cli_args(parser)
args = parser.parse_args(
[
"--dataset-name",
"random-mm",
"--num-prompts",
"3",
"--random-input-len",
"32",
"--random-output-len",
"1",
"--random-mm-base-items-per-request",
"1",
"--random-mm-num-mm-items-range-ratio",
"0",
"--random-mm-limit-mm-per-prompt",
'{"image": 1}',
"--random-mm-bucket-config",
"{(224, 224, 1): 1.0}",
]
)
requests = get_requests(args, hf_tokenizer)
assert len(requests) == 3
assert all(isinstance(r, SampleRequest) for r in requests)
# mm-processor drives llm.chat, so prompts must be chat-formatted with
# multimodal content attached.
for req in requests:
assert isinstance(req.prompt, list)
assert req.prompt[0]["role"] == "user"
assert any(item.get("type") == "image_url" for item in req.prompt[0]["content"])
@@ -1,68 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from pathlib import Path
import pytest
from transformers import AutoTokenizer, PreTrainedTokenizerBase
from vllm.benchmarks.datasets import CustomDataset
from vllm.benchmarks.datasets.create_txt_slices_dataset import create_txt_slices_jsonl
@pytest.fixture(scope="session")
def hf_tokenizer() -> PreTrainedTokenizerBase:
# Use a small, commonly available tokenizer
return AutoTokenizer.from_pretrained("openai-community/gpt2")
text_content = """
Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor
incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud
exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.
Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat
nulla pariatur. Excepteur sint occaecat cupidatat non proident,
sunt in culpa qui officia deserunt mollit anim id est laborum.
"""
@pytest.mark.benchmark
def test_create_txt_slices_jsonl(
hf_tokenizer: PreTrainedTokenizerBase, tmp_path: Path
) -> None:
"""Test that create_txt_slices_jsonl produces valid JSONL for CustomDataset."""
txt_path = tmp_path / "input.txt"
jsonl_path = tmp_path / "input.txt.jsonl"
txt_path.write_text(text_content)
create_txt_slices_jsonl(
input_path=str(txt_path),
output_path=str(jsonl_path),
tokenizer_name="openai-community/gpt2",
num_prompts=10,
input_len=10,
output_len=10,
)
# Verify the JSONL file is valid and has the expected structure
records = [json.loads(line) for line in jsonl_path.read_text().splitlines()]
assert len(records) == 10
for record in records:
assert "prompt" in record
assert "output_tokens" in record
assert isinstance(record["prompt"], str)
assert record["output_tokens"] == 10
# Verify the JSONL file can be loaded by CustomDataset
dataset = CustomDataset(dataset_path=str(jsonl_path))
samples = dataset.sample(
tokenizer=hf_tokenizer,
num_requests=10,
output_len=10,
skip_chat_template=True,
)
assert len(samples) == 10
assert all(sample.expected_output_len == 10 for sample in samples)
@@ -1,71 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from torch import nn
from vllm.compilation.decorators import support_torch_compile
from vllm.config import (
CompilationConfig,
CompilationMode,
VllmConfig,
set_current_vllm_config,
)
from vllm.forward_context import set_forward_context
from vllm.model_executor.models.qwen3_vl import (
Qwen3VLForConditionalGeneration,
)
from vllm.sequence import IntermediateTensors
@support_torch_compile(
dynamic_arg_dims={"x": 0, "deepstack_input_embeds": 0},
)
class _DeepStackDecoder(nn.Module):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
super().__init__()
def forward(
self,
x: torch.Tensor,
deepstack_input_embeds: IntermediateTensors | None,
) -> torch.Tensor:
if deepstack_input_embeds is not None:
x = x + deepstack_input_embeds["deepstack_input_embeds_0"]
return x
def _run_contract(
deepstack_buffer: Qwen3VLForConditionalGeneration,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
config = VllmConfig(
compilation_config=CompilationConfig(mode=CompilationMode.VLLM_COMPILE)
)
with set_current_vllm_config(config):
model = _DeepStackDecoder(vllm_config=config).eval().cuda()
x = torch.arange(8, dtype=torch.float32, device="cuda").reshape(2, 4)
with torch.inference_mode(), set_forward_context(None, vllm_config=config):
empty_payload = deepstack_buffer._get_deepstack_input_embeds(num_tokens=2)
first = model(x, empty_payload)
payload = torch.full((1, 2, 4), 7.0, device="cuda")
deepstack_buffer._set_deepstack_input_embeds(payload)
active_payload = deepstack_buffer._get_deepstack_input_embeds(num_tokens=2)
assert active_payload is not None
second = model(x, active_payload)
return x.cpu(), first.cpu(), second.cpu()
def test_qwen3_vl_deepstack_payload_after_empty_compile(monkeypatch):
monkeypatch.setenv("VLLM_DISABLE_COMPILE_CACHE", "1")
deepstack_buffer = object.__new__(Qwen3VLForConditionalGeneration)
deepstack_buffer.deepstack_input_embeds = [torch.zeros(2, 4, device="cuda")]
deepstack_buffer.deepstack_input_embeds_num_tokens = 0
deepstack_buffer.deepstack_num_level = 1
x, first, second = _run_contract(deepstack_buffer)
torch.testing.assert_close(first, x)
torch.testing.assert_close(second, x + 7)
-727
View File
@@ -1,727 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import random
import pytest
import ray
import torch
import torch.distributed as dist
from vllm.distributed.communication_op import tensor_model_parallel_all_reduce # noqa
from vllm.distributed.device_communicators import custom_all_reduce as car
from vllm.distributed.parallel_state import get_tp_group, graph_capture
from vllm.platforms import current_platform
from ..utils import (
ensure_model_parallel_initialized,
init_test_distributed_environment,
multi_process_parallel,
)
random.seed(42)
test_sizes = [random.randint(1024, 2048 * 1024) for _ in range(8)]
for i, v in enumerate(test_sizes):
test_sizes[i] -= v % 8
def _bf16_ulps(a: torch.Tensor, b: torch.Tensor) -> int:
"""Largest distance between two BF16 tensors in units in the last place."""
def ordered(t: torch.Tensor) -> torch.Tensor:
bits = t.view(torch.int16).int() & 0xFFFF
return torch.where(bits >= 0x8000, 0x8000 - bits, bits)
return int((ordered(a) - ordered(b)).abs().max())
def _unfused_all_reduce_mhc(peers, residual, post, comb, pre, weight, eps):
"""All-reduce, mHC post, collapse and RMSNorm in the fused kernel's FP32
order, rounding to BF16 where the unfused path returns BF16."""
reduced = peers[0].float()
for peer in peers[1:]:
reduced = reduced + peer.float()
reduced = reduced.bfloat16().float()
output = torch.empty_like(residual)
collapse = torch.zeros_like(reduced)
for target in range(4):
mixed = reduced * post[:, target : target + 1]
for source in range(4):
mixed = torch.addcmul(
mixed, residual[:, source].float(), comb[:, source, target, None]
)
output[:, target] = mixed.bfloat16()
collapse = torch.addcmul(
collapse, output[:, target].float(), pre[:, target : target + 1]
)
prenorm = collapse.bfloat16().float()
inv_rms = torch.rsqrt(prenorm.square().mean(-1, keepdim=True) + eps)
return output, (prenorm * inv_rms * weight.float()).bfloat16()
@ray.remote(num_gpus=1, max_calls=1)
def _all_reduce_mhc(monkeypatch, tp_size, pp_size, rank, distributed_init_port):
from vllm.models.deepseek_v41.nvidia.ops.cute_dsl import AllReduceMHC
with monkeypatch.context() as m:
m.delenv("CUDA_VISIBLE_DEVICES", raising=False)
device = torch.device(f"cuda:{rank}")
torch.accelerator.set_device_index(device)
init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port)
ensure_model_parallel_initialized(tp_size, pp_size)
op = AllReduceMHC(
hidden_size=5120, hc_mult=4, max_num_tokens=16, top_k=6, device=device
)
def check_eager_and_replayed(fused, check, halved, doubled):
check(fused())
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
for _ in range(5):
fused()
captured = fused()
for _ in range(20):
graph.replay()
check(captured)
# Replays must pick up in-place input changes.
halved.mul_(0.5)
doubled.mul_(2)
graph.replay()
check(captured)
def mhc_inputs(n):
torch.manual_seed(123)
residual = torch.randn(n, 4, 5120, device=device, dtype=torch.bfloat16)
post = torch.rand(n, 4, device=device)
comb = torch.randn(n, 4, 4, device=device) * 0.1
pre = torch.rand(n, 4, device=device)
weight = torch.randn(5120, device=device, dtype=torch.bfloat16)
return residual, post, comb, pre, weight
def run(n):
torch.manual_seed(42 + rank)
x = torch.randn(n, 5120, device=device, dtype=torch.bfloat16)
# Packed +0/-0 pairs collide with the Lamport sentinel.
x[:, :16] = 0
x[:, 9:16:2] = -0.0
residual, post, comb, pre, weight = mhc_inputs(n)
def fused():
return op(x, residual, post, comb, pre, weight, 1e-6)
def check(outputs):
peers = get_tp_group().all_gather(x, dim=0).view(tp_size, n, 5120)
output, normalized = _unfused_all_reduce_mhc(
peers, residual, post, comb, pre, weight, 1e-6
)
# The mixed hc streams match bit for bit. The RMSNorm sums the
# squares in another order and uses an approximate rsqrt.
assert torch.equal(outputs[0], output)
assert _bf16_ulps(outputs[1], normalized) <= 1
check_eager_and_replayed(fused, check, x, residual)
def run_finalize(n):
torch.manual_seed(7 + rank)
# A padded permuted GEMM2 buffer, like the MoE's.
rows = n * 6 + 5
gemm2 = torch.randn(rows, 5120, device=device, dtype=torch.bfloat16)
permuted = torch.randperm(rows, device=device)[: n * 6].view(n, 6).int()
# A route to an expert this rank does not hold.
permuted[0, -1] = -1
weights = torch.rand(n, 6, device=device)
shared = torch.randn(n, 5120, device=device, dtype=torch.bfloat16)
residual, post, comb, pre, weight = mhc_inputs(n)
def fused():
return op.finalize(
gemm2,
weights,
permuted,
shared,
residual,
post,
comb,
pre,
weight,
1e-6,
)
def check(outputs):
# One FP32 FMA per route in route order, the shared add, one
# rounding, then the plain path: only the finalize differs.
acc = torch.zeros(n, 5120, device=device)
for k in range(6):
valid = (permuted[:, k] >= 0).unsqueeze(-1)
rows_k = gemm2[permuted[:, k].clamp_min(0).long()].float()
torch.addcmul(
acc,
torch.where(valid, rows_k, 0.0),
torch.where(valid, weights[:, k : k + 1], 0.0),
out=acc,
)
x = (acc + shared.float()).bfloat16()
expected = op(x, residual, post, comb, pre, weight, 1e-6)
assert torch.equal(outputs[0], expected[0])
assert torch.equal(outputs[1], expected[1])
check_eager_and_replayed(fused, check, gemm2, shared)
# Changing shapes cover shrinking and growing batches in one mailbox.
for n in (1, 6, 12, 8, 16, 3, 5, 2, 4, 1):
run(n)
run_finalize(n)
@pytest.mark.skipif(
not current_platform.is_device_capability_family(100), reason="Requires SM100"
)
def test_all_reduce_mhc_matches_unfused_path(monkeypatch):
if torch.accelerator.device_count() < 4:
pytest.skip("Requires four GPUs with NVLink multicast")
multi_process_parallel(monkeypatch, 4, 1, _all_reduce_mhc)
@pytest.mark.parametrize(
("dtype", "expected"),
[
(torch.float32, True),
(torch.float16, True),
(torch.bfloat16, True),
(torch.int8, False),
(torch.float8_e4m3fn, False),
],
)
def test_custom_allreduce_filters_dtype(
dtype: torch.dtype,
expected: bool,
) -> None:
communicator = car.CustomAllreduce.__new__(car.CustomAllreduce)
communicator.disabled = False
communicator._ptr = 0
communicator.world_size = 2
communicator.max_size = 1024
assert communicator.should_custom_ar(torch.empty(16, dtype=dtype)) is expected
@pytest.mark.parametrize("batch_invariant", [False, True])
def test_custom_allreduce_size_gate_ignored_under_batch_invariance(
batch_invariant: bool,
) -> None:
"""Batch invariance must not switch backends based on the input size."""
communicator = car.CustomAllreduce.__new__(car.CustomAllreduce)
communicator.disabled = False
communicator.world_size = 2
communicator.max_size = 1024
communicator.batch_invariant = batch_invariant
communicator._ptr = 0
oversized = torch.empty(1024, dtype=torch.float16)
assert communicator.should_custom_ar(oversized) is batch_invariant
@pytest.mark.parametrize("batch_invariant", [False, True])
def test_custom_reduce_scatter_disabled_under_batch_invariance(
monkeypatch, batch_invariant: bool
) -> None:
"""Reduce-scatter stays on one backend under batch invariance."""
monkeypatch.setattr(car.current_platform, "is_cuda", lambda: True)
communicator = car.CustomAllreduce.__new__(car.CustomAllreduce)
communicator.disabled = False
communicator.world_size = 2
communicator.fully_connected = True
communicator.mnnvl_only = False
communicator.mnnvl_multicast_ptr = 0
communicator.max_reduce_scatter_size = 1024
communicator.max_mnnvl_reduce_scatter_size = 1024
communicator.batch_invariant = batch_invariant
in_range = torch.empty(64, dtype=torch.float16)
assert communicator.should_custom_reduce_scatter(in_range) is not batch_invariant
@pytest.mark.parametrize(
("major", "local_multicast", "expected"),
[
(8, True, False),
(9, True, False),
(10, False, False),
(10, True, True),
],
)
def test_cross_node_mnnvl_gate_checks_generation_and_multicast(
monkeypatch,
major,
local_multicast,
expected,
):
def has_device_capability(capability, device_id):
assert capability == 100
assert device_id == 3
return major >= 10
monkeypatch.setattr(
car.current_platform,
"has_device_capability",
has_device_capability,
)
monkeypatch.setattr(
car,
"_has_local_multicast_support",
lambda _device: local_multicast,
)
monkeypatch.setattr(car.dist, "all_reduce", lambda *_args, **_kwargs: None)
assert car._group_can_attempt_mnnvl(object(), torch.device("cuda:3")) is expected
def test_cross_node_mnnvl_gate_requires_support_on_every_rank(monkeypatch):
monkeypatch.setattr(
car.current_platform,
"has_device_capability",
lambda *_args: True,
)
monkeypatch.setattr(
car,
"_has_local_multicast_support",
lambda _device: True,
)
def report_unsupported_peer(support, **_kwargs):
support.zero_()
monkeypatch.setattr(car.dist, "all_reduce", report_unsupported_peer)
assert not car._group_can_attempt_mnnvl(object(), torch.device("cuda:0"))
def test_local_multicast_support_rejects_non_cuda(monkeypatch):
monkeypatch.setattr(car.current_platform, "is_cuda", lambda: False)
assert not car._has_local_multicast_support(torch.device("cuda:0"))
@pytest.mark.parametrize(
("world_size", "device_capability", "local_multicast", "expected"),
[
(2, (10, 0), True, True),
(4, (10, 3), True, True),
(8, (10, 0), True, True),
(8, (10, 3), True, True),
(6, (10, 3), True, False),
(8, (10, 1), True, False),
(8, (9, 0), True, False),
(8, (10, 3), False, False),
],
)
def test_mnnvl_multimem_reduce_scatter_platform_gate(
monkeypatch,
world_size,
device_capability,
local_multicast,
expected,
):
def is_device_capability(capability, device_id):
assert capability in ((10, 0), (10, 3))
assert device_id == 3
return device_capability == capability
monkeypatch.setattr(
car.current_platform,
"is_device_capability",
is_device_capability,
)
monkeypatch.setattr(
car,
"_has_local_multicast_support",
lambda _device: local_multicast,
)
supported = car._supports_mnnvl_multimem_reduce_scatter(
torch.device("cuda:3"), world_size
)
assert supported is expected
@pytest.mark.parametrize(
(
"message_bytes",
"multimem_ptr",
"multimem_initialized",
"batch_invariant",
"expected",
),
[
(16 * 1024 * 1024, 1, True, False, "mnnvl_lamport"),
(16 * 1024 * 1024 + 128, 1, True, False, "mnnvl_multimem"),
(64 * 1024 * 1024, 1, True, False, "mnnvl_multimem"),
(64 * 1024 * 1024 + 128, 1, True, False, None),
(32 * 1024 * 1024, 0, True, False, None),
(32 * 1024 * 1024, 0, False, False, "mnnvl_multimem"),
(8 * 1024 * 1024, 1, True, True, "mnnvl_lamport"),
(32 * 1024 * 1024, 1, True, True, None),
],
)
@pytest.mark.parametrize("world_size", [2, 4, 8])
def test_mnnvl_reduce_scatter_backend_gate(
monkeypatch,
world_size,
message_bytes,
multimem_ptr,
multimem_initialized,
batch_invariant,
expected,
):
monkeypatch.setattr(car.current_platform, "is_cuda", lambda: True)
monkeypatch.setattr(car.envs, "VLLM_BATCH_INVARIANT", batch_invariant)
communicator = car.CustomAllreduce.__new__(car.CustomAllreduce)
communicator.disabled = False
communicator._ptr = 0
communicator.world_size = world_size
communicator.mnnvl_only = False
communicator.fully_connected = True
communicator.mnnvl_multicast_ptr = 1
communicator.mnnvl_multimem_rs_supported = True
communicator.mnnvl_multimem_rs_initialized = multimem_initialized
communicator.mnnvl_multimem_rs_multicast_ptr = multimem_ptr
communicator.max_mnnvl_reduce_scatter_size = 16 * 1024 * 1024
communicator.max_mnnvl_multimem_reduce_scatter_size = 64 * 1024 * 1024
communicator.max_reduce_scatter_size = 16 * 1024 * 1024
inp = torch.empty(
(world_size, message_bytes // torch.bfloat16.itemsize // world_size),
dtype=torch.bfloat16,
)
assert inp.nbytes == message_bytes
assert communicator._select_reduce_scatter_backend(inp) == expected
assert communicator.should_custom_reduce_scatter(inp) is (expected is not None)
assert communicator.should_mnnvl_multimem_reduce_scatter(inp) is (
expected == "mnnvl_multimem"
)
def test_mnnvl_multimem_reduce_scatter_skips_rendezvous_after_peer_alloc_failure(
monkeypatch,
):
events = []
class FakeSymmMem:
@staticmethod
def empty(*_args, **_kwargs):
events.append("empty")
return torch.empty(1, dtype=torch.uint8)
@staticmethod
def rendezvous(*_args, **_kwargs):
events.append("rendezvous")
return None
def report_peer_allocation_failure(group_value, **_kwargs):
events.append("all_reduce")
assert group_value.item() == 1
group_value.zero_()
monkeypatch.setattr(car, "torch_symm_mem", FakeSymmMem)
monkeypatch.setattr(car.ops, "meta_size", lambda: 128)
monkeypatch.setattr(car.dist, "all_reduce", report_peer_allocation_failure)
warnings = []
monkeypatch.setattr(
car.logger,
"warning_once",
lambda message, *_args, **_kwargs: warnings.append(message),
)
communicator = car.CustomAllreduce.__new__(car.CustomAllreduce)
communicator.disabled = True
communicator._ptr = 0
communicator.group = object()
communicator.device = torch.device("cuda:0")
communicator.max_mnnvl_multimem_reduce_scatter_size = 64 * 1024 * 1024
communicator.mnnvl_multimem_rs_supported = True
communicator.mnnvl_multimem_rs_initialized = False
communicator.mnnvl_multimem_rs_buffer = None
communicator.mnnvl_multimem_rs_multicast_ptr = 0
communicator._init_mnnvl_multimem_reduce_scatter_buffer()
assert events == ["empty", "all_reduce"]
assert communicator.mnnvl_multimem_rs_initialized
assert communicator.mnnvl_multimem_rs_buffer is None
assert communicator.mnnvl_multimem_rs_multicast_ptr == 0
assert warnings == [
"MNNVL multimem reduce-scatter symmetric-memory allocation "
"failed on at least one rank; falling back to NCCL."
]
def test_mnnvl_multimem_reduce_scatter_warns_on_rendezvous_failure(monkeypatch):
events = []
class FakeSymmMem:
@staticmethod
def empty(*_args, **_kwargs):
events.append("empty")
return torch.empty(1, dtype=torch.uint8)
@staticmethod
def rendezvous(*_args, **_kwargs):
events.append("rendezvous")
raise RuntimeError("rendezvous failed")
def preserve_local_result(_group_value, **_kwargs):
events.append("all_reduce")
warnings = []
monkeypatch.setattr(car, "torch_symm_mem", FakeSymmMem)
monkeypatch.setattr(car.ops, "meta_size", lambda: 128)
monkeypatch.setattr(car.dist, "all_reduce", preserve_local_result)
monkeypatch.setattr(
car.logger,
"warning_once",
lambda message, *_args, **_kwargs: warnings.append(message),
)
communicator = car.CustomAllreduce.__new__(car.CustomAllreduce)
communicator.disabled = True
communicator._ptr = 0
communicator.group = type("Group", (), {"group_name": "test"})()
communicator.device = torch.device("cuda:0")
communicator.max_mnnvl_multimem_reduce_scatter_size = 64 * 1024 * 1024
communicator.mnnvl_multimem_rs_supported = True
communicator.mnnvl_multimem_rs_initialized = False
communicator.mnnvl_multimem_rs_buffer = None
communicator.mnnvl_multimem_rs_multicast_ptr = 0
communicator._init_mnnvl_multimem_reduce_scatter_buffer()
assert events == ["empty", "all_reduce", "rendezvous", "all_reduce"]
assert communicator.mnnvl_multimem_rs_initialized
assert communicator.mnnvl_multimem_rs_buffer is None
assert communicator.mnnvl_multimem_rs_multicast_ptr == 0
assert warnings == [
"MNNVL multimem reduce-scatter symmetric-memory rendezvous "
"failed on at least one rank; falling back to NCCL."
]
def test_mnnvl_multimem_reduce_scatter_initializes_signals(monkeypatch):
events = []
buffers = []
class FakeHandle:
multicast_ptr = 0x3000
class FakeSymmMem:
@staticmethod
def empty(size, **_kwargs):
events.append(("empty", size))
buffer = torch.ones(size, dtype=torch.uint8)
buffers.append(buffer)
return buffer
@staticmethod
def rendezvous(*_args, **_kwargs):
events.append(("rendezvous", None))
return FakeHandle()
def preserve_local_result(*_args, **_kwargs):
events.append(("all_reduce", None))
monkeypatch.setattr(car, "torch_symm_mem", FakeSymmMem)
monkeypatch.setattr(car.ops, "meta_size", lambda: 128)
monkeypatch.setattr(
car.torch.accelerator,
"synchronize",
lambda: events.append(("synchronize", None)),
)
monkeypatch.setattr(car.dist, "all_reduce", preserve_local_result)
communicator = car.CustomAllreduce.__new__(car.CustomAllreduce)
communicator.disabled = True
communicator._ptr = 0
communicator.group = type("Group", (), {"group_name": "test"})()
communicator.device = torch.device("cpu")
communicator.max_mnnvl_multimem_reduce_scatter_size = 129
communicator.mnnvl_multimem_rs_supported = True
communicator.mnnvl_multimem_rs_initialized = False
communicator.mnnvl_multimem_rs_buffer = None
communicator.mnnvl_multimem_rs_multicast_ptr = 0
communicator._init_mnnvl_multimem_reduce_scatter_buffer()
assert events == [
("empty", 257),
("all_reduce", None),
("rendezvous", None),
("synchronize", None),
("all_reduce", None),
]
assert torch.all(buffers[0][:128] == 0)
assert torch.all(buffers[0][128:] == 1)
assert communicator.mnnvl_multimem_rs_buffer_size == 129
assert communicator.mnnvl_multimem_rs_local_ptr == buffers[0].data_ptr() + 128
assert communicator.mnnvl_multimem_rs_multicast_ptr == 0x3080
@ray.remote(num_gpus=1, max_calls=1)
def graph_allreduce(
monkeypatch: pytest.MonkeyPatch,
tp_size,
pp_size,
rank,
distributed_init_port,
):
with monkeypatch.context() as m:
m.delenv("CUDA_VISIBLE_DEVICES", raising=False)
m.delenv("HIP_VISIBLE_DEVICES", raising=False)
device = torch.device(f"cuda:{rank}")
torch.accelerator.set_device_index(device)
init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port)
ensure_model_parallel_initialized(tp_size, pp_size)
group = get_tp_group().device_group
# A small all_reduce for warmup.
# this is needed because device communicators might be created lazily
# (e.g. NCCL). This will ensure that the communicator is initialized
# before any communication happens, so that this group can be used for
# graph capture immediately.
data = torch.zeros(1)
data = data.to(device=device)
torch.distributed.all_reduce(data, group=group)
torch.accelerator.synchronize()
del data
# we use the first group to communicate once
# and the second group to communicate twice
# and so on
# this is used to demonstrate that each group can
# communicate independently
num_communication = rank // tp_size + 1
for sz in test_sizes:
for dtype in [torch.float32, torch.float16, torch.bfloat16]:
with graph_capture(device=device) as graph_capture_context:
# use integers so result matches NCCL exactly
device_idx = torch.accelerator.current_device_index()
inp1 = torch.randint(1, 16, (sz,), dtype=dtype, device=device_idx)
inp2 = torch.randint(1, 16, (sz,), dtype=dtype, device=device_idx)
torch.accelerator.synchronize()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph, stream=graph_capture_context.stream):
for i in range(num_communication):
out1 = tensor_model_parallel_all_reduce(inp1)
# the input buffer is immediately modified to test
# synchronization
dist.all_reduce(inp1, group=group)
out2 = tensor_model_parallel_all_reduce(inp2)
dist.all_reduce(inp2, group=group)
graph.replay()
torch.testing.assert_close(out1, inp1)
torch.testing.assert_close(out2, inp2)
@ray.remote(num_gpus=1, max_calls=1)
def eager_allreduce(
monkeypatch: pytest.MonkeyPatch,
tp_size,
pp_size,
rank,
distributed_init_port,
):
with monkeypatch.context() as m:
m.delenv("CUDA_VISIBLE_DEVICES", raising=False)
m.delenv("HIP_VISIBLE_DEVICES", raising=False)
device = torch.device(f"cuda:{rank}")
torch.accelerator.set_device_index(device)
init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port)
# we use the first group to communicate once
# and the second group to communicate twice
# and so on
# this is used to demonstrate that each group can
# communicate independently
num_communication = rank // tp_size + 1
sz = 1024
fa = get_tp_group().device_communicator.ca_comm
inp = torch.ones(sz, dtype=torch.float32, device=device)
out = inp
for _ in range(num_communication):
out = fa.all_reduce(out, registered=False)
torch.testing.assert_close(out, inp * (tp_size**num_communication))
inp = torch.ones(sz * 4, dtype=torch.bfloat16, device=device)
out = inp
for _ in range(num_communication):
out = fa.all_reduce(out, registered=False)
torch.testing.assert_close(out, inp * (tp_size**num_communication))
@ray.remote(num_gpus=1, max_calls=1)
def chunked_allreduce(
monkeypatch: pytest.MonkeyPatch,
tp_size,
pp_size,
rank,
distributed_init_port,
):
"""Inputs above max_size are reduced in chunks and match the fixed-order
reference bitwise, both eagerly and inside a CUDA graph."""
with monkeypatch.context() as m:
m.delenv("CUDA_VISIBLE_DEVICES", raising=False)
m.delenv("HIP_VISIBLE_DEVICES", raising=False)
m.setenv("VLLM_CUSTOM_ALLREDUCE_ALGO", "1stage")
device = torch.device(f"cuda:{rank}")
torch.accelerator.set_device_index(device)
init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port)
ensure_model_parallel_initialized(tp_size, pp_size)
group = get_tp_group().device_group
fa = get_tp_group().device_communicator.ca_comm
# Two full chunks plus a partial one.
dtype = torch.bfloat16
chunk_numel = fa.max_size // dtype.itemsize
inp = torch.randn(2 * chunk_numel + 4096, device=device).to(dtype)
gathered = [torch.empty_like(inp) for _ in range(tp_size)]
dist.all_gather(gathered, inp, group=group)
ref = gathered[0].float()
for peer in gathered[1:]:
ref = ref + peer.float()
ref = ref.to(torch.bfloat16)
out = fa.all_reduce(inp, registered=False)
assert torch.equal(out, ref)
# Weak-contiguous but not C-contiguous: a transposed matrix.
inp_t, ref_t = inp.view(-1, 4096).t(), ref.view(-1, 4096).t()
assert torch.equal(fa.all_reduce(inp_t, registered=False), ref_t)
with fa.capture():
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
out = fa.all_reduce(inp, registered=True)
graph.replay()
assert torch.equal(out, ref)
@pytest.mark.parametrize("tp_size", [2, 4])
def test_custom_allreduce_chunked(monkeypatch: pytest.MonkeyPatch, tp_size):
if tp_size > torch.accelerator.device_count():
pytest.skip("Not enough GPUs to run the test.")
multi_process_parallel(monkeypatch, tp_size, 1, chunked_allreduce)
@pytest.mark.parametrize("tp_size", [2])
@pytest.mark.parametrize("pipeline_parallel_size", [1, 2])
@pytest.mark.parametrize("test_target", [eager_allreduce, graph_allreduce])
def test_custom_allreduce(
monkeypatch: pytest.MonkeyPatch,
tp_size,
pipeline_parallel_size,
test_target,
):
world_size = tp_size * pipeline_parallel_size
if world_size > torch.accelerator.device_count():
pytest.skip("Not enough GPUs to run the test.")
multi_process_parallel(monkeypatch, tp_size, pipeline_parallel_size, test_target)
File diff suppressed because it is too large Load Diff
-883
View File
@@ -1,883 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Engram tables shared by co-located DP replicas.
Four GPUs on one node split the hash heads four ways, whether as DEP
(TP1 x DP4, the single-node MoE layout) or TP2 x DP2. A lookup only returns
the right rows if the ids of every replica are gathered, each rank looks up
its own heads, and each replica trades those tokens back for the heads its
peers own.
With shared host storage, DP replicas instead map the same TP slice and
prefetch only their own tokens without DP gathers.
"""
from itertools import product
from pathlib import Path
from tempfile import TemporaryDirectory
from threading import Event
from types import SimpleNamespace
import pytest
import torch
import torch.multiprocessing as mp
from vllm.config import (
EngramConfig,
VllmConfig,
get_current_vllm_config_or_none,
set_current_vllm_config,
)
from vllm.config.load import LoadConfig
from vllm.config.parallel import ParallelConfig
from vllm.config.scheduler import SchedulerConfig
from vllm.distributed import cleanup_dist_env_and_memory, parallel_state
from vllm.distributed.parallel_state import (
get_engram_dp_group,
get_engram_dp_size,
init_distributed_environment,
initialize_model_parallel,
)
from vllm.forward_context import set_forward_context
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
dequant_mxfp8_to_bf16,
)
from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import (
convert_swizzled_to_linear,
)
from vllm.models.deepseek_v41.common import engram as engram_ops
from vllm.models.deepseek_v41.common.engram import (
Engram,
EngramBatch,
EngramLayout,
ParallelEngramEmbedding,
engram_head_shard_rank,
gather_engram_hashes,
)
from vllm.platforms import current_platform
from vllm.utils.network_utils import get_open_port
from vllm.utils.system_utils import update_environment_variables
WORLD_SIZE = 4
DIM, BLOCK = 64, 32
# Prime-sized buckets, as the model's layout builds them.
HEAD_SIZES = (97, 101, 103, 107, 109, 113, 127, 131)
def _token_counts(dp_size: int) -> tuple[tuple[int, ...], ...]:
"""Even loads, then uneven ones with an idle replica to pad around."""
return ((8,) * dp_size, (7, 0, 5, 1)[:dp_size], (7,) * dp_size)
def _full_table(rows: int) -> tuple[torch.Tensor, torch.Tensor]:
"""The unsharded table, identical on every rank."""
torch.manual_seed(0)
weight = (torch.randn(rows, DIM) * 4).to(torch.float8_e4m3fn)
scale_inv = torch.randint(120, 134, (rows, DIM // BLOCK), dtype=torch.uint8)
return weight, scale_inv
def _make_ids(head_sizes: tuple[int, ...], num_tokens: int, seed: int) -> torch.Tensor:
"""Ids of each head, drawn from that head's own bucket range."""
torch.manual_seed(seed)
offset = 0
columns = []
for size in head_sizes:
columns.append(torch.randint(offset, offset + size, (num_tokens, 1)))
offset += size
return torch.cat(columns, dim=1).to(torch.int32).cuda()
def _reference(weight, scale_inv, ids) -> torch.Tensor:
"""Dequantized lookup against the whole table."""
valid = ids >= 0
safe_ids = ids.clamp_min(0).long()
values = torch.nn.functional.embedding(safe_ids, weight).float()
scales = torch.nn.functional.embedding(safe_ids, scale_inv)
scales = (scales.to(torch.int32) << 23).view(torch.float32)
values = values.unflatten(-1, (-1, BLOCK)) * scales.unsqueeze(-1)
return values.flatten(-2).to(torch.bfloat16).masked_fill(~valid.unsqueeze(-1), 0)
@pytest.mark.parametrize(
"architecture,options,edp_ranks,etp_ranks",
[
("DeepseekV41ForCausalLM", {}, [0, 2], [0, 1]),
("Qwen4ExpForCausalLM", {}, None, [0, 1]),
(
"Qwen4ExpForCausalLM",
{"embedding_across_dp": True},
None,
[0, 1, 2, 3],
),
("Qwen4ExpForConditionalGeneration", {}, None, [0, 1]),
],
)
def test_engram_group_creation_honors_model_and_options(
monkeypatch, architecture, options, edp_ranks, etp_ranks
):
config = SimpleNamespace(
model_config=SimpleNamespace(architecture=architecture, is_moe=False),
parallel_config=ParallelConfig(
tensor_parallel_size=2,
data_parallel_size=2,
distributed_executor_backend="mp",
),
engram_config=EngramConfig(**options),
)
for name in (
"_TP",
"_ETP",
"_ENGRAM_DP",
"_DCP",
"_PCP",
"_PP",
"_DP",
"_EP",
"_EPLB",
):
monkeypatch.setattr(parallel_state, name, None)
monkeypatch.setattr(parallel_state, "_WORLD", SimpleNamespace(local_rank=0))
monkeypatch.setattr(torch.distributed, "is_initialized", lambda: True)
monkeypatch.setattr(torch.distributed, "get_world_size", lambda: 4)
monkeypatch.setattr(torch.distributed, "get_rank", lambda: 0)
monkeypatch.setattr(parallel_state, "get_node_count", lambda: 1)
def make_group(group_ranks, *args, **kwargs):
ranks = next(ranks for ranks in group_ranks if 0 in ranks)
return SimpleNamespace(ranks=ranks, rank_in_group=0, world_size=len(ranks))
monkeypatch.setattr(parallel_state, "init_model_parallel_group", make_group)
with set_current_vllm_config(config):
initialize_model_parallel(tensor_model_parallel_size=2, backend="nccl")
group = get_engram_dp_group()
assert (group.ranks if group is not None else None) == edp_ranks
assert parallel_state.get_etp_group().ranks == etp_ranks
def _worker(rank: int, tp_size: int, port: int) -> None:
torch.accelerator.set_device_index(rank)
update_environment_variables(
{
"RANK": str(rank),
"LOCAL_RANK": str(rank),
"WORLD_SIZE": str(WORLD_SIZE),
"MASTER_ADDR": "localhost",
"MASTER_PORT": str(port),
}
)
dp_size = WORLD_SIZE // tp_size
dp_rank, tp_rank = divmod(rank, tp_size)
vllm_config = VllmConfig(
parallel_config=ParallelConfig(
tensor_parallel_size=tp_size,
data_parallel_size=dp_size,
data_parallel_rank=dp_rank,
),
scheduler_config=SchedulerConfig(
max_model_len=8,
is_encoder_decoder=False,
max_num_batched_tokens=8,
max_num_seqs=8,
),
)
# Supply the resolved Engram settings without constructing a full model.
vllm_config.engram_config = EngramConfig()
vllm_config.model_config = SimpleNamespace(
architecture="DeepseekV41ForCausalLM",
is_moe=True,
sleep_mode_offload_cudagraph=False,
)
try:
init_distributed_environment()
with set_current_vllm_config(vllm_config):
initialize_model_parallel(tensor_model_parallel_size=tp_size)
assert get_engram_dp_size() == dp_size
assert get_engram_dp_group().ranks == [
tp_rank + replica * tp_size for replica in range(dp_size)
]
assert engram_head_shard_rank() == tp_rank * dp_size + dp_rank
for failure in ("create", "open", "mmap", "register", "pinned", None):
_check_shared_storage_lifetime_and_failures(failure)
# Reuse the processes and communication groups across storage modes.
for (cpu_offload, dp_shared_memory), n_heads, sequence_parallel in product(
[(False, False), (True, False), (True, True)],
[8, 7],
[False, True] if tp_size > 1 else [False],
):
_check_table(
vllm_config,
cpu_offload,
dp_shared_memory,
n_heads,
sequence_parallel,
)
if tp_size == 1:
_check_fp8_batch(vllm_config)
finally:
cleanup_dist_env_and_memory()
def _check_fp8_batch(vllm_config):
"""MXFP8 staging matches the tables across peer/all-to-all slots, idle
replicas and graph replays."""
from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphCapture
dp_rank = vllm_config.parallel_config.data_parallel_rank
layers = []
for index in range(2):
weight, scale = _full_table(sum(HEAD_SIZES))
weight = (weight.float() * (1 - 2 * index)).to(torch.float8_e4m3fn)
table = ParallelEngramEmbedding(
sum(HEAD_SIZES), DIM, HEAD_SIZES, cpu_offload=True
)
table.weight.weight_loader(table.weight, weight)
table.weight_scale_inv.weight_loader(table.weight_scale_inv, scale)
staged = torch.empty(0, device="cuda")
reference = (weight.cuda(), scale.cuda())
layers.append(SimpleNamespace(embed_tokens=table, staged_rows=staged))
layers[-1].layer_hash_index, layers[-1].reference = index, reference
batch = EngramBatch(layers, 257)
def check(ids):
tokens = ids.shape[0]
for layer in layers:
x = batch.wkv_input(layer, tokens)
sf = convert_swizzled_to_linear(x.scale, tokens, x.data.shape[1], BLOCK)
actual = dequant_mxfp8_to_bf16(x.data, sf).unflatten(-1, (-1, DIM))
weight, scale = layer.reference
expected = _reference(weight, scale, ids[:, layer.layer_hash_index])
torch.testing.assert_close(actual, expected, atol=0, rtol=0)
for counts in ((7, 0, 5, 1), (127,) * 4, (128,) * 4, (257, 0, 129, 1)):
tokens = counts[dp_rank]
seed = 700 + 10 * dp_rank
ids = torch.stack([_make_ids(HEAD_SIZES, tokens, seed + i) for i in (0, 1)], 1)
with set_forward_context(
None,
vllm_config,
num_tokens=tokens,
num_tokens_across_dp=torch.tensor(counts, dtype=torch.int32),
):
batch.prepare_embeddings(ids)
check(ids)
for capture in ("full", "breakable") if len(set(counts)) == 1 else ():
warmup = torch.cuda.Stream()
warmup.wait_stream(torch.cuda.current_stream())
if capture == "full":
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph, stream=warmup):
batch.prepare_embeddings(ids)
else:
graph = BreakableCUDAGraphCapture()
with torch.cuda.stream(warmup), graph:
batch.prepare_embeddings(ids)
torch.cuda.current_stream().wait_stream(warmup)
for step in (2, 3):
replay_ids = _make_ids(HEAD_SIZES, tokens, seed + step)
ids.copy_(torch.stack([replay_ids] * 2, 1))
graph.replay()
check(ids)
del graph
def _check_shared_storage_lifetime_and_failures(failure):
"""One peer's failure must roll back every mapping/registration, without hangs."""
import gc
import weakref
group = get_engram_dp_group()
runtime = torch.cuda.cudart()
make_file = engram_ops.tempfile.NamedTemporaryFile
map_file = engram_ops.mmap.mmap
paths, mappings, registrations, unregistrations = [], [], [], []
failed_rank = 0 if failure == "create" else 1
def fail():
raise OSError(f"injected {failure} failure")
def temporary_file(*args, **kwargs):
if failure == "create":
fail()
file = make_file(*args, **kwargs)
paths.append(Path(file.name))
return file
def open_file(*args, **kwargs):
if failure == "open" and group.rank_in_group == failed_rank:
fail()
return open(*args, **kwargs)
def mapping(*args, **kwargs):
if failure == "mmap" and group.rank_in_group == failed_rank:
fail()
result = map_file(*args, **kwargs)
mappings.append(weakref.ref(result))
return result
def register(pointer, size, flags):
if failure == "register" and group.rank_in_group == failed_rank:
return SimpleNamespace(value=1)
result = runtime.cudaHostRegister(pointer, size, flags)
assert result.value == 0
registrations.append(pointer)
return result
def unregister(pointer):
unregistrations.append(pointer)
result = runtime.cudaHostUnregister(pointer)
assert result.value == 0
return result
with pytest.MonkeyPatch.context() as patch:
patch.setattr(
engram_ops,
"tempfile",
SimpleNamespace(NamedTemporaryFile=temporary_file),
)
patch.setattr(engram_ops, "open", open_file, raising=False)
patch.setattr(
engram_ops,
"mmap",
SimpleNamespace(mmap=mapping, MAP_SHARED=engram_ops.mmap.MAP_SHARED),
)
patch.setattr(
torch.cuda,
"cudart",
lambda: SimpleNamespace(
cudaHostRegister=register,
cudaHostUnregister=unregister,
),
)
if failure == "pinned" and group.rank_in_group == failed_rank:
patch.setattr(torch.Tensor, "is_pinned", lambda self: False)
if failure is not None:
stage = "cudaHostRegister" if failure in ("register", "pinned") else failure
with pytest.raises(RuntimeError, match=f"EDP rank {failed_rank}.*{stage}"):
engram_ops.DPSharedEngramStorage(128, DIM, BLOCK, group)
assert unregistrations == registrations
else:
storage = engram_ops.DPSharedEngramStorage(128, DIM, BLOCK, group)
assert all(not path.exists() for path in paths)
storage_ref = weakref.ref(storage)
parameter = torch.nn.Parameter(storage.weight, requires_grad=False)
views = storage.get_views(parameter, storage.weight_scale_inv)
del storage
gc.collect()
assert storage_ref() is None
assert not unregistrations
del parameter
gc.collect()
assert not unregistrations # UVA views still retain the CPU allocation.
del views
gc.collect()
assert unregistrations == registrations
for ref in mappings:
mapped = ref()
assert mapped is None or mapped.closed
assert all(not path.exists() for path in paths)
def _check_table(
vllm_config, cpu_offload, dp_shared_memory, n_heads, sequence_parallel
):
parallel = vllm_config.parallel_config
dp_size, tp_size = parallel.data_parallel_size, parallel.tensor_parallel_size
dp_rank = parallel.data_parallel_rank
tp_rank = engram_ops.get_tensor_model_parallel_rank()
multithread = (
dp_shared_memory and n_heads == 7 and sequence_parallel == (tp_size > 1)
)
load_config = (
LoadConfig(
load_format="safetensors",
use_tqdm_on_load=False,
model_loader_extra_config={
"enable_multithread_load": True,
"num_threads": 2,
},
)
if multithread
else vllm_config.load_config
)
with pytest.MonkeyPatch.context() as m:
head_sizes = HEAD_SIZES[:n_heads]
rows = sum(head_sizes)
config = SimpleNamespace(
hidden_size=DIM,
hc_mult=2,
rms_norm_eps=1e-6,
engram_layer_ids=[0],
engram_num_embeddings=[rows],
engram_max_ngram_size=2,
engram_n_heads=n_heads,
engram_head_dim=DIM,
engram_compressed_vocab_size=32,
engram_pad_token_id=0,
engram_vocab_size=HEAD_SIZES[0],
)
with set_current_vllm_config(vllm_config):
weight, scale_inv = _full_table(rows)
layout = EngramLayout(config)
# Stub configuration inputs, not Engram's initialization or execution.
component_config = SimpleNamespace(
engram_config=EngramConfig(
cpu_offload=cpu_offload,
dp_shared_memory=dp_shared_memory,
),
scheduler_config=vllm_config.scheduler_config,
load_config=load_config,
)
with m.context() as init_patch, torch.device("cuda"):
init_patch.setattr(
engram_ops, "get_current_vllm_config", lambda: component_config
)
engram = Engram(
config,
quant_config=None,
layout=layout,
layer_hash_index=0,
use_sequence_parallel=sequence_parallel,
prefix="model.layers.0.engram",
)
layer = engram.embed_tokens
if multithread:
_load_shared_multithread(layer, weight, scale_inv, load_config)
else:
# Only the leader has valid checkpoint payload in shared mode.
loader_weight = weight if not dp_shared_memory or dp_rank == 0 else None
loader_scale = (
scale_inv if not dp_shared_memory or dp_rank == 0 else None
)
layer.weight.weight_loader(layer.weight, loader_weight)
layer.weight_scale_inv.weight_loader(
layer.weight_scale_inv, loader_scale
)
if dp_shared_memory:
assert layer.dp_size == 1
assert layer.head_start == tp_rank * layer.part_n_hash_cols
assert layer.weight.is_pinned() and layer.weight_scale_inv.is_pinned()
def unexpected_collective(*args, **kwargs):
raise AssertionError("shared lookup must not use DP collectives")
m.setattr(get_engram_dp_group(), "all_gather", unexpected_collective)
token_counts = _token_counts(dp_size)
# Production forward runs after the construction-time config has exited.
assert get_current_vllm_config_or_none() is None
for batch_idx, counts in enumerate(token_counts):
num_tokens = counts[dp_rank]
# Ids differ per replica but must agree across its TP ranks.
ids = _make_ids(head_sizes, num_tokens, seed=100 + dp_rank)
if batch_idx == 2 and dp_rank == 1:
ids.fill_(engram_ops.DEAD_ID)
with set_forward_context(
None,
vllm_config,
num_tokens=num_tokens,
num_tokens_across_dp=torch.tensor(counts, dtype=torch.int32),
):
gathered = gather_engram_hashes(
ids, dp_shared_memory=layer.dp_shared_memory
)
expected_tokens = (
num_tokens if dp_shared_memory else max(counts) * dp_size
)
assert gathered.shape[0] == expected_tokens
engram.prepare_embeddings(gathered)
looked_up = engram.embed(ids)
standalone = layer(ids)
expected = _reference(weight.cuda(), scale_inv.cuda(), ids)
torch.testing.assert_close(standalone, expected, atol=0, rtol=0)
if sequence_parallel:
# SP hands back only this rank's token shard, zero-padded.
chunk = -(-num_tokens // tp_size)
expected = torch.nn.functional.pad(
expected, (0, 0, 0, 0, 0, chunk * tp_size - num_tokens)
)
expected = expected[tp_rank * chunk : (tp_rank + 1) * chunk]
torch.testing.assert_close(looked_up, expected, atol=0, rtol=0)
if (
cpu_offload
and n_heads == 7
and not sequence_parallel
and (not dp_shared_memory or tp_size == 1)
):
_check_dummy_hash_model_forward(
vllm_config,
engram,
head_sizes,
weight,
scale_inv,
dp_shared_memory,
)
replay = n_heads == 7 and (
(dp_shared_memory and sequence_parallel == (tp_size > 1))
or (
cpu_offload
and not dp_shared_memory
and sequence_parallel == (tp_size > 1)
)
)
if replay:
captures = ("full", "breakable")
for capture in captures:
_check_prefetch_replay(
engram,
head_sizes,
weight,
scale_inv,
dp_shared_memory,
capture,
)
if dp_shared_memory and n_heads == 7 and sequence_parallel == (tp_size > 1):
with m.context() as storage_patch:
storage_patch.setattr(layer.weight, "data", layer.weight.data.clone())
with pytest.raises(RuntimeError, match="storage must not be replaced"):
layer(ids)
def _check_dummy_hash_model_forward(
vllm_config, engram, head_sizes, weight, scales, dp_shared_memory
):
"""A metadata-less replica joins DP sharding but skips a shared lookup."""
from vllm.models.deepseek_v41.common.engram import NgramHashState
from vllm.models.deepseek_v41.nvidia import model as model_ops
dp_rank = vllm_config.parallel_config.data_parallel_rank
if dp_shared_memory and dp_rank != 1:
return
tokens = 4
ids = _make_ids(head_sizes, tokens, seed=300 + dp_rank)
state = NgramHashState.__new__(NgramHashState)
torch.nn.Module.__init__(state)
state.multipliers = torch.empty(1, 2, dtype=torch.int64, device="cuda")
state.primes = torch.empty(1, 1, len(head_sizes), dtype=torch.int64, device="cuda")
state.ensure_cache = lambda: True
# Hash arithmetic is covered separately; keep the real model's branch and
# NgramHashState.dummy_hashes, plus real preparation/lookup collectives.
state.forward = lambda *args: ids.unsqueeze(1)
class Decoder(SimpleNamespace):
# Engram hashes and the mask stay the last positional arguments;
# **kwargs absorbs the decoder's keyword-only flags.
def __call__(self, hidden, positions, input_ids, *args, **kwargs):
hashes, keep = args[-2:]
if dp_shared_memory and dp_rank == 1:
assert hashes is None and keep is None
output = hidden.new_zeros(
tokens, len(head_sizes), DIM, dtype=torch.bfloat16
)
else:
assert hashes is not None and keep is not None
assert bool(keep.all()) == (dp_rank != 1)
if dp_rank == 1:
assert torch.all(hashes == engram_ops.DEAD_ID)
output = engram.embed(hashes[:, 0])
# Trailing None is previous_aux; this stub captures no aux states.
return output, hidden, hidden, hidden, hidden, None
model = model_ops.DeepseekV4Model.__new__(model_ops.DeepseekV4Model)
torch.nn.Module.__init__(model)
model.parallel_config = vllm_config.parallel_config
model.use_native_mega_moe = False
model.use_sequence_parallel = False
model.fuse_mhc_all_reduce = False
model.engram_batch = None
model.engram_hash = state
model.engram_swa_prefix = "swa"
model.engram_dp_shared_memory = dp_shared_memory
model.layers = [Decoder(engram=engram)]
model.start_layer = 0
model.end_layer = 1
model.decoder_replay_start = model.end_layer
model.decoder_replay_layers = None
model.aux_hidden_state_layers = ()
metadata = (
None
if dp_rank == 1
else {
"swa": SimpleNamespace(
query_start_loc=None, slot_mapping=None, block_table=None
)
}
)
with pytest.MonkeyPatch.context() as patch:
patch.setattr(
model_ops,
"get_pp_group",
lambda: SimpleNamespace(
is_first_rank=True,
is_last_rank=False,
),
)
patch.setattr(model_ops, "mhc_post_tilelang", lambda hidden, *args: hidden)
with set_forward_context(
metadata,
vllm_config,
num_tokens=tokens,
num_tokens_across_dp=torch.full(
(vllm_config.parallel_config.data_parallel_size,),
tokens,
dtype=torch.int32,
),
):
output = model(
torch.arange(tokens, device="cuda"),
torch.arange(tokens, device="cuda"),
intermediate_tensors=None,
inputs_embeds=torch.zeros(tokens, DIM, device="cuda"),
lookback_token_ids=torch.full((1, 1), -1, device="cuda"),
)["hidden_states"]
expected_ids = torch.full_like(ids, engram_ops.DEAD_ID) if dp_rank == 1 else ids
expected = _reference(weight.cuda(), scales.cuda(), expected_ids)
torch.testing.assert_close(output, expected, atol=0, rtol=0)
def _load_shared_multithread(layer, weight, scale_inv, load_config):
"""Opposite shard completion orders must leave every shared parameter ready."""
from safetensors.torch import save_file
from vllm.model_executor.model_loader import weight_utils
from vllm.model_executor.model_loader.default_loader import DefaultModelLoader
tensors = {
"weight": weight,
"weight_scale_inv": scale_inv.view(torch.float8_e8m0fnu),
}
order = list(tensors)
if get_engram_dp_group().rank_in_group % 2:
order.reverse()
first_loaded = Event()
original_load = weight_utils.load_file
def load_file(path, **kwargs):
if Path(path).stem != order[0]:
assert first_loaded.wait(timeout=30), "First shared weight load stalled"
return original_load(path, **kwargs)
with TemporaryDirectory() as directory, pytest.MonkeyPatch.context() as m:
for name, tensor in tensors.items():
save_file({name: tensor}, str(Path(directory) / f"{name}.safetensors"))
m.setattr(weight_utils, "load_file", load_file)
loader = DefaultModelLoader(load_config)
source = DefaultModelLoader.Source(directory, revision=None)
loaded = []
for name, tensor in loader._get_weights_iterator(source):
param = getattr(layer, name)
param.weight_loader(param, tensor)
loaded.append(name)
first_loaded.set()
assert loaded == order
# Check before any further collective could mask incomplete shared writes.
for name, tensor in tensors.items():
param = getattr(layer, name)
expected = tensor.narrow(0, param.engram_vocab_start, param.shape[0])
torch.testing.assert_close(
param.view(torch.uint8), expected.view(torch.uint8), atol=0, rtol=0
)
def _check_prefetch_replay(
engram, head_sizes, weight, scale_inv, dp_shared_memory, capture
):
"""DP-sharded and shared prefetches must replay with new IDs."""
from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphCapture
ids = _make_ids(head_sizes, 8, seed=501)
tp_rank = engram_ops.get_tensor_model_parallel_rank()
tokens = 8 // engram.embed_tokens.tp_size if engram.use_sequence_parallel else 8
output = torch.empty(
tokens, len(head_sizes), DIM, device="cuda", dtype=torch.bfloat16
)
def step():
gathered = gather_engram_hashes(ids.clone(), dp_shared_memory=dp_shared_memory)
engram.prepare_embeddings(gathered)
output.copy_(engram.embed(ids))
with pytest.MonkeyPatch.context() as patch:
patch.setattr(engram_ops, "engram_gathered_num_tokens", lambda: 8)
warmup = torch.cuda.Stream()
warmup.wait_stream(torch.cuda.current_stream())
with torch.cuda.stream(warmup):
step()
if capture == "breakable":
graph = BreakableCUDAGraphCapture()
with torch.cuda.stream(warmup), graph:
step()
assert graph.num_eager_breaks == 2
else:
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph, stream=warmup):
step()
torch.cuda.current_stream().wait_stream(warmup)
for iteration in range(2):
ids.copy_(_make_ids(head_sizes, 8, seed=600 + iteration))
graph.replay()
expected = _reference(weight.cuda(), scale_inv.cuda(), ids)
if engram.use_sequence_parallel:
expected = expected[tp_rank * tokens : (tp_rank + 1) * tokens]
torch.testing.assert_close(output, expected, atol=0, rtol=0)
torch.accelerator.synchronize()
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA required")
@pytest.mark.parametrize("tp_size", [1, 2])
def test_engram_table_shared_across_dp_replicas(tp_size, monkeypatch):
"""Check all storage modes with one process launch per TP/DP topology."""
if torch.accelerator.device_count() < WORLD_SIZE:
pytest.skip(f"Need {WORLD_SIZE} GPUs to run the test.")
monkeypatch.setenv("VLLM_USE_BREAKABLE_CUDAGRAPH", "1")
mp.spawn(_worker, args=(tp_size, get_open_port()), nprocs=WORLD_SIZE)
cleanup_dist_env_and_memory()
@pytest.mark.parametrize(
"cpu_offload,dp_size,error",
[
(False, 4, "requires cpu_offload=True"),
(True, 1, "effective Engram DP size is 1"),
],
)
def test_engram_dp_shared_memory_runtime_requirements(
monkeypatch, cpu_offload, dp_size, error
):
"""Report unmet storage/topology requirements before allocating CUDA memory."""
monkeypatch.setattr(engram_ops, "get_engram_dp_size", lambda: dp_size)
with pytest.raises(ValueError, match=error):
ParallelEngramEmbedding(
sum(HEAD_SIZES),
DIM,
HEAD_SIZES,
cpu_offload=cpu_offload,
dp_shared_memory=True,
)
@pytest.mark.parametrize("query_fails", [False, True])
def test_engram_tables_too_large_for_shm_are_not_shared(monkeypatch, query_fails):
"""A /dev/shm that is too small or cannot be queried must fall back, and the
leader must still broadcast that decision so its peers do not hang."""
broadcasts = []
def broadcast(obj):
broadcasts.append(obj)
return obj
def query(*args, **kwargs):
raise OSError("statvfs failed")
monkeypatch.setattr(engram_ops, "get_engram_dp_size", lambda: 2)
monkeypatch.setattr(
engram_ops,
"get_engram_dp_group",
lambda: SimpleNamespace(rank_in_group=0, broadcast_object=broadcast),
)
if query_fails:
monkeypatch.setattr(engram_ops, "check_shm_free_space", query)
layout = SimpleNamespace(num_embeddings=(1 << 50,), head_dim=DIM)
assert not engram_ops.can_share_engram_tables(layout)
assert len(broadcasts) == 1 and broadcasts[0] is not None
@pytest.mark.parametrize(
"node_ids,dp_size,replica_size,expected",
[
([0] * 4, 1, 4, 1),
([0] * 4, 4, 1, 4),
([0] * 4, 2, 2, 2),
([0] * 8 + [1] * 8, 8, 2, 4),
([0] * 6 + [1] * 6, 4, 1, 2),
([0] * 3 + [1] * 4, 7, 1, 1),
([0] * 8 + [1] * 8 + [2] * 8, 8, 3, 1),
([0, 1] * 4, 4, 2, 1),
([0] * 3 + [1] * 5, 4, 2, 1),
],
)
def test_engram_dp_shard_size_respects_node_boundaries(
node_ids, dp_size, replica_size, expected, monkeypatch
):
"""Sharing must fall back when replicas or node rank ranges do not align."""
monkeypatch.setattr(parallel_state, "get_node_count", lambda: len(set(node_ids)))
monkeypatch.setattr(
parallel_state, "get_world_group", lambda: SimpleNamespace(cpu_group=None)
)
monkeypatch.setattr(
parallel_state,
"in_the_same_node_as",
lambda group, src: [node == node_ids[src] for node in node_ids],
)
assert (
parallel_state._engram_dp_shard_size(len(node_ids), dp_size, replica_size)
== expected
)
@pytest.mark.parametrize("num_tokens", [0, 2, 4, 5])
def test_engram_hash_padding_has_no_valid_rows(num_tokens, monkeypatch):
"""Idle/padded slots must not query row zero; oversized batches are rejected."""
monkeypatch.setattr(engram_ops, "engram_gathered_num_tokens", lambda: 4)
monkeypatch.setattr(
engram_ops,
"get_engram_dp_group",
lambda: SimpleNamespace(all_gather=lambda ids, dim: torch.cat([ids, ids], dim)),
)
ids = torch.full((num_tokens, 2, 3), 17, dtype=torch.int32)
if num_tokens > 4:
with pytest.raises(ValueError, match="exceeds the DP token slot"):
gather_engram_hashes(ids)
return
gathered = gather_engram_hashes(ids).reshape(2, 4, 2, 3)
for replica in gathered:
torch.testing.assert_close(replica[:num_tokens], ids)
assert torch.all(replica[num_tokens:] == engram_ops.DEAD_ID)
@pytest.mark.parametrize("tp_size", [1, 2])
@pytest.mark.parametrize("uniform", [False, True])
def test_engram_hash_padding_ignores_other_nodes(tp_size, uniform, monkeypatch):
"""Two four-GPU nodes pad only to their own maximum, including TP x DP."""
group_size = 4 // tp_size
counts = (
[4] * (8 // tp_size)
if uniform
else [4096] + [2] * (group_size - 1) + [1, 3] + [0] * (group_size - 2)
)
monkeypatch.setattr(
engram_ops,
"get_forward_context",
lambda: SimpleNamespace(
dp_metadata=SimpleNamespace(num_tokens_across_dp_cpu=torch.tensor(counts))
),
)
for dp_rank in range(8 // tp_size):
slot = 4 if uniform else (4096 if dp_rank < group_size else 3)
ids = torch.full((counts[dp_rank], 2, 3), 17, dtype=torch.int32)
monkeypatch.setattr(
engram_ops,
"get_dp_group",
lambda rank=dp_rank: SimpleNamespace(rank_in_group=rank),
)
monkeypatch.setattr(
engram_ops,
"get_engram_dp_group",
lambda rank=dp_rank: SimpleNamespace(
rank_in_group=rank % group_size,
world_size=group_size,
all_gather=lambda tensor, dim: torch.cat([tensor] * group_size, dim),
),
)
gathered = gather_engram_hashes(ids)
expected = ids.new_full((slot, 2, 3), engram_ops.DEAD_ID)
expected[: ids.shape[0]] = ids
torch.testing.assert_close(
gathered, expected.repeat(group_size, 1, 1), msg=f"DP rank {dp_rank}"
)
-355
View File
@@ -1,355 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
from types import SimpleNamespace
import lm_eval
import pytest
import torch
from transformers import DeepseekV4Config
from tests.utils import large_gpu_mark
from vllm.distributed.eplb.eplb_state import EplbState
from vllm.models.deepseek_v4.common.eplb_util import dspark_draft_supports_eplb
from vllm.platforms import current_platform
from vllm.v1.worker.gpu.eplb_utils import (
EPLBController,
draft_model_supports_eplb,
)
def get_model_args(
model_name: str,
spec_model_name: str | None,
spec_method: str,
tp_size: int,
model_max_len: int,
num_speculative_tokens: int = 1,
use_async: bool = True,
) -> dict:
speculative_config = {
"method": spec_method,
"model": spec_model_name,
"num_speculative_tokens": num_speculative_tokens,
"max_model_len": model_max_len,
}
if spec_method == "dspark":
speculative_config["model"] = model_name
speculative_config["draft_sample_method"] = "probabilistic"
eplb_config = {
"num_redundant_experts": tp_size,
"window_size": 128,
"step_interval": 1024,
"log_balancedness": False,
"use_async": use_async,
}
model_args = {
"pretrained": model_name,
"dtype": "auto",
"add_bos_token": True,
"tensor_parallel_size": tp_size,
"gpu_memory_utilization": 0.7,
"speculative_config": speculative_config,
"enable_expert_parallel": True,
"eplb_config": eplb_config,
"enable_eplb": True,
"max_model_len": model_max_len,
}
if spec_method == "dspark":
model_args["trust_remote_code"] = True
return model_args
pytestmark = pytest.mark.skipif(
current_platform.is_rocm(),
reason="EPLB with Spec Decode is a work in progress on ROCm.",
)
class FakeEplbState:
instances: list[FakeEplbState] = []
def __init__(self, parallel_config, device: torch.device):
self.add_model_calls: list[tuple[object, object, str | None]] = []
FakeEplbState.instances.append(self)
def add_model(
self,
model: object,
model_config: object,
*,
model_name: str | None = None,
) -> None:
self.add_model_calls.append((model, model_config, model_name))
def _make_dsv4_dspark_hf_config() -> DeepseekV4Config:
return DeepseekV4Config(
architectures=["DeepseekV4ForCausalLM"],
hidden_size=128,
num_hidden_layers=2,
n_routed_experts=8,
num_experts_per_tok=2,
num_hash_layers=0,
n_shared_experts=1,
moe_intermediate_size=128,
hc_mult=1,
hc_eps=1e-5,
rms_norm_eps=1e-5,
dspark_target_layer_ids=[0],
dspark_markov_rank=8,
index_topk=4,
head_dim=64,
num_attention_heads=4,
vocab_size=256,
n_mtp_layers=2,
enable_confidence_head=False,
layer_types=["sliding_attention"] * 2,
)
@pytest.fixture
def dspark_vllm_config():
hf_config = _make_dsv4_dspark_hf_config()
model_config = SimpleNamespace(
dtype=torch.bfloat16, hf_config=hf_config, model="dspark"
)
return SimpleNamespace(
model_config=model_config,
quant_config=None,
kernel_config=SimpleNamespace(moe_backend="deep_gemm_mega_moe"),
parallel_config=SimpleNamespace(
pipeline_parallel_size=1,
tensor_parallel_size=1,
data_parallel_size=1,
enable_expert_parallel=True,
enable_eplb=True,
enable_elastic_ep=False,
eplb_config=SimpleNamespace(num_redundant_experts=4),
),
scheduler_config=SimpleNamespace(max_num_batched_tokens=4),
compilation_config=SimpleNamespace(static_forward_context={}),
speculative_config=SimpleNamespace(
method="dspark",
draft_model_config=model_config,
),
)
def _make_moe_topology(
*,
num_routed_experts: int = 8,
num_redundant_experts: int = 4,
num_expert_groups: int = 1,
) -> SimpleNamespace:
num_physical_experts = num_routed_experts + num_redundant_experts
return SimpleNamespace(
num_routed_experts=num_routed_experts,
num_redundant_experts=num_redundant_experts,
num_physical_experts=num_physical_experts,
num_logical_experts=num_routed_experts,
num_expert_groups=num_expert_groups,
)
def test_eplb_state_accepts_matching_dsv4_draft_and_target():
draft = _make_moe_topology()
target = _make_moe_topology()
EplbState.assert_confs_equal(draft, target)
def test_eplb_state_rejects_mismatched_dsv4_draft_redundant_experts():
draft = _make_moe_topology(num_redundant_experts=0)
target = _make_moe_topology(num_redundant_experts=4)
with pytest.raises(RuntimeError, match="mismatch"):
EplbState.assert_confs_equal(draft, target)
@pytest.mark.parametrize(
("model_type", "expected"),
[
("deepseek_v4", True),
("deepseek_v41", False),
],
)
def test_dspark_draft_supports_eplb(
dspark_vllm_config, model_type: str, expected: bool
):
draft_model_config = dspark_vllm_config.speculative_config.draft_model_config
draft_model_config.hf_config.model_type = model_type
assert dspark_draft_supports_eplb(draft_model_config) is expected
def test_draft_model_supports_eplb_requires_draft_moe(dspark_vllm_config):
assert not draft_model_supports_eplb(
dspark_vllm_config.speculative_config,
None,
)
@pytest.mark.parametrize(
"model_type,load_dummy_weights,expected_registered",
[
("deepseek_v4", False, True),
("deepseek_v4", True, False),
("deepseek_v41", False, False),
],
)
def test_eplb_dspark_draft_registration(
dspark_vllm_config,
monkeypatch: pytest.MonkeyPatch,
model_type,
load_dummy_weights: bool,
expected_registered: bool,
):
"""Register V4 drafts; skip incompatible V4.1 drafts and dummy weights."""
FakeEplbState.instances.clear()
monkeypatch.setattr(
"vllm.v1.worker.gpu.eplb_utils.EplbState",
FakeEplbState,
)
monkeypatch.setattr(
"vllm.v1.worker.gpu.eplb_utils.get_mixture_of_experts_model",
lambda model: model,
)
draft_model_config = dspark_vllm_config.speculative_config.draft_model_config
draft_model_config.hf_config.model_type = model_type
draft = SimpleNamespace()
controller = EPLBController(dspark_vllm_config.parallel_config, torch.device("cpu"))
controller.prepare_load()
speculator = SimpleNamespace(model=draft, eplb_state=None)
def set_eplb_state(state) -> None:
speculator.eplb_state = state
speculator.set_eplb_state = set_eplb_state
registered = controller.maybe_register_speculator(
speculator,
dspark_vllm_config.speculative_config,
load_dummy_weights=load_dummy_weights,
)
assert registered is expected_registered
assert controller.state is not None
if expected_registered:
assert controller.state.add_model_calls == [
(draft, draft_model_config, "dspark (draft)")
]
assert speculator.eplb_state is controller.state
else:
assert controller.state.add_model_calls == []
assert speculator.eplb_state is None
@pytest.mark.parametrize(
"model_setup",
[
pytest.param(
("mtp", "Qwen/Qwen3-Next-80B-A3B-Instruct", None, 4, 0.86, 1),
marks=large_gpu_mark(min_gb=80),
),
pytest.param(
(
"eagle",
"meta-llama/Llama-4-Scout-17B-16E-Instruct",
"morgendave/EAGLE-Llama-4-Scout-17B-16E-Instruct",
4,
0.92,
1,
),
marks=pytest.mark.skip(reason="Skipping due to CI OOM issues"),
),
pytest.param(
(
"dspark",
"deepseek-ai/DeepSeek-V4-Flash-DSpark",
None,
4,
0.95,
7,
),
marks=large_gpu_mark(min_gb=80),
),
],
ids=["qwen3_next_mtp", "llama4_eagle", "dsv4_dspark"],
)
def test_eplb_spec_decode(
monkeypatch: pytest.MonkeyPatch,
model_setup: tuple[str, str, str | None, int, float, int],
):
"""Test the correctness of EPLB speculative decoding with GSM8K dataset.
Applicable to MoE models with mtp, eagle, or dspark spec decode.
"""
(
method,
model_name,
spec_model_name,
tp_size,
expected_gsm8k_value,
num_speculative_tokens,
) = model_setup
if method == "dspark" and not current_platform.is_device_capability_family(100):
pytest.skip("DSV4 DSpark EPLB requires SM100 for MegaMoE")
TASK = "gsm8k"
FILTER = "exact_match,strict-match"
RTOL = 0.03
model_args = get_model_args(
model_name=model_name,
spec_model_name=spec_model_name,
spec_method=method,
tp_size=tp_size,
model_max_len=4096,
num_speculative_tokens=num_speculative_tokens,
)
results = lm_eval.simple_evaluate(
model="vllm",
model_args=model_args,
tasks=TASK,
batch_size=64,
num_fewshot=8,
)
measured_value = results["results"][TASK][FILTER]
assert (
measured_value - RTOL < expected_gsm8k_value
and measured_value + RTOL > expected_gsm8k_value
), f"Expected: {expected_gsm8k_value} | Measured: {measured_value}"
@large_gpu_mark(min_gb=80)
def test_eplb_spec_decode_qwen3_next_mtp_async() -> None:
"""Ensure async EPLB works with MTP speculative decoding for Qwen3-Next."""
TASK = "gsm8k"
FILTER = "exact_match,strict-match"
RTOL = 0.03
expected_gsm8k_value = 0.86
model_args = get_model_args(
model_name="Qwen/Qwen3-Next-80B-A3B-Instruct",
spec_model_name=None,
spec_method="mtp",
tp_size=4,
model_max_len=4096,
use_async=True,
)
results = lm_eval.simple_evaluate(
model="vllm",
model_args=model_args,
tasks=TASK,
batch_size=64,
num_fewshot=8,
)
measured_value = results["results"][TASK][FILTER]
assert (
measured_value - RTOL < expected_gsm8k_value
and measured_value + RTOL > expected_gsm8k_value
), f"Expected: {expected_gsm8k_value} | Measured: {measured_value}"
File diff suppressed because it is too large Load Diff
@@ -1,81 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import weakref
import pytest
from vllm import LLM, PoolingParams
from vllm.exceptions import VLLMValidationError
MODEL_NAME = "intfloat/multilingual-e5-small"
PROMPTS = [
"Hello, my name is",
"The president of the United States is",
"The capital of France is",
"The future of AI is",
]
TOKEN_IDS = [
# Using ID={0, 1, 2, 3} results in NaN values,
# so we add this offset of 1000
[1000],
[1000, 1001],
[1000, 1002, 1001],
[1000, 1003, 1001, 1002],
]
@pytest.fixture(scope="module")
def llm(vllm_runner):
with vllm_runner(
MODEL_NAME,
max_model_len=None,
max_num_batched_tokens=32768,
tensor_parallel_size=1,
gpu_memory_utilization=0.75,
enforce_eager=True,
seed=0,
enable_chunked_prefill=None,
) as runner:
# pytest caches yielded fixtures until after teardown, so use a proxy to
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
yield weakref.proxy(runner.llm)
@pytest.mark.skip_global_cleanup
def test_multiple_pooling_params(llm: LLM):
pooling_params = [
PoolingParams(),
PoolingParams(),
PoolingParams(),
PoolingParams(),
]
# Multiple PoolingParams should be matched with each prompt
outputs = llm.encode(PROMPTS, pooling_params=pooling_params, pooling_task="embed")
assert len(PROMPTS) == len(outputs)
# Exception raised, if the size of params does not match the size of prompts
with pytest.raises(VLLMValidationError):
outputs = llm.encode(
PROMPTS, pooling_params=pooling_params[:3], pooling_task="embed"
)
# Single PoolingParams should be applied to every prompt
single_pooling_params = PoolingParams()
outputs = llm.encode(
PROMPTS, pooling_params=single_pooling_params, pooling_task="embed"
)
assert len(PROMPTS) == len(outputs)
# pooling_params is None, default params should be applied
outputs = llm.encode(PROMPTS, pooling_params=None, pooling_task="embed")
assert len(PROMPTS) == len(outputs)
def test_right_side_truncation(llm: LLM):
# Embeddings models should truncate the end of the prompt
tokenizer = llm.get_tokenizer()
assert tokenizer.truncation_side == "right"
@@ -1,102 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import weakref
from unittest import mock
import pytest
from vllm import PoolingParams
MODEL_NAME = "intfloat/multilingual-e5-small"
@pytest.fixture(scope="module")
def llm(vllm_runner):
with vllm_runner(
MODEL_NAME,
max_model_len=None,
max_num_seqs=2, # small to trigger tiling
tensor_parallel_size=1,
gpu_memory_utilization=0.75,
enforce_eager=True,
seed=0,
enable_chunked_prefill=None,
) as runner:
# pytest caches yielded fixtures until after teardown, so use a proxy to
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
yield weakref.proxy(runner.llm)
@pytest.mark.skip_global_cleanup
def test_tiling_engine_basic(llm):
"""Basic test with a small number of prompts (less than max_num_seqs).
No tiling should be triggered, but the engine still processes correctly.
"""
prompts = ["Hello", "World"]
outputs = llm.encode(prompts, pooling_task="embed")
assert len(outputs) == len(prompts)
@pytest.mark.skip_global_cleanup
def test_tiling_engine_many_requests(llm):
"""Test with a large number of prompts that exceeds max_num_seqs.
This verifies that _run_tiling_engine correctly chunks requests,
processes all of them, and returns outputs in the correct order.
"""
num_prompts = 10
prompts = [f"Prompt {i}" for i in range(num_prompts)]
outputs = llm.encode(prompts, pooling_task="embed")
assert len(outputs) == num_prompts
@pytest.mark.skip_global_cleanup
def test_tiling_engine_with_pooling_params(llm):
"""Test the tiling engine when different PoolingParams are provided.
The engine must handle a list of params that matches the number of prompts.
"""
num_prompts = 10
prompts = [f"Prompt {i}" for i in range(num_prompts)]
pooling_params = [PoolingParams() for _ in range(num_prompts)]
outputs = llm.encode(prompts, pooling_params=pooling_params, pooling_task="embed")
assert len(outputs) == num_prompts
# Single PoolingParams shared across all prompts
single_param = PoolingParams()
outputs = llm.encode(prompts, pooling_params=single_param, pooling_task="embed")
assert len(outputs) == num_prompts
# None PoolingParams should fall back to default
outputs = llm.encode(prompts, pooling_params=None, pooling_task="embed")
assert len(outputs) == num_prompts
@pytest.mark.skip_global_cleanup
def test_tiling_engine_abort_on_exception(llm):
"""Test that abort_request IS called with the correct arguments when an
exception occurs inside the engine's step() loop.
"""
prompts = ["Prompt 0", "Prompt 1", "Prompt 2"]
# Mock the step method to throw an exception on the second call
original_step = llm.llm_engine.step
call_count = 0
def mocked_step():
nonlocal call_count
call_count += 1
if call_count == 2:
raise RuntimeError("Simulated engine error")
return original_step()
with mock.patch.object(llm.llm_engine, "step", side_effect=mocked_step):
# We expect an exception to be raised from encode
with mock.patch.object(llm.llm_engine, "abort_request") as mock_abort: # noqa: SIM117
with pytest.raises(RuntimeError, match="Simulated engine error"):
llm.encode(prompts, pooling_task="embed")
args, kwargs = mock_abort.call_args
request_ids = args[0]
assert isinstance(request_ids, list)
assert len(request_ids) > 0
@@ -1,99 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any
import openai
import pytest
import pytest_asyncio
from tests.utils import RemoteOpenAIServer
MODEL_NAME = "sentence-transformers/all-MiniLM-L12-v2"
max_model_len = 128
input = """Immerse yourself in the enchanting chronicle of calculus, a
mathematical domain that has radically transformed our comprehension of
change and motion. Despite its roots in ancient civilizations, the
formal birth of calculus predominantly occurred in the 17th century,
primarily under the influential guidance of Sir Isaac Newton and Gottfried
Wilhelm Leibniz. The earliest traces of calculus concepts are found in
ancient Greek mathematics,most notably in the works of Eudoxus and
Archimedes, around 300 BCE. They utilized the 'method of exhaustion'—a
technique for computing areas and volumes through the use of finite sums.
This methodology laid crucial foundational work for integral calculus.
In the 17th century, both Newton and Leibniz independently pioneered
calculus, each contributing unique perspectives that would shape this new
field."""
@pytest.fixture(scope="module")
def server():
args = [
"--runner",
"pooling",
"--dtype",
"bfloat16",
"--enforce-eager",
"--max-model-len",
str(max_model_len),
]
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
yield remote_server
@pytest_asyncio.fixture
async def client(server):
async with server.get_async_client() as async_client:
yield async_client
@pytest.mark.asyncio
async def test_smaller_truncation_size(client: openai.AsyncOpenAI):
truncation_size = 10
kwargs: dict[str, Any] = {
"model": MODEL_NAME,
"input": input,
"truncate_prompt_tokens": truncation_size,
}
response = await client.post(path="embeddings", cast_to=object, body={**kwargs})
assert response["usage"]["prompt_tokens"] == truncation_size
@pytest.mark.asyncio
async def test_bigger_truncation_size(client: openai.AsyncOpenAI):
truncation_size = max_model_len + 1
kwargs: dict[str, Any] = {
"model": MODEL_NAME,
"input": input,
"truncate_prompt_tokens": truncation_size,
}
with pytest.raises(openai.BadRequestError) as err:
await client.post(path="embeddings", cast_to=object, body={**kwargs})
assert err.value.status_code == 400
error_details = err.value.response.json()["error"]
assert error_details["type"] == "BadRequestError"
expected_message = (
"truncate_prompt_tokens value is "
"greater than max_model_len."
" Please request a smaller truncation size."
)
assert error_details["message"] == expected_message
@pytest.mark.asyncio
async def test_max_truncation_size(client: openai.AsyncOpenAI):
truncation_size = -1
kwargs: dict[str, Any] = {
"model": MODEL_NAME,
"input": input,
"truncate_prompt_tokens": truncation_size,
}
response = await client.post(path="embeddings", cast_to=object, body={**kwargs})
assert response["usage"]["prompt_tokens"] == max_model_len
@@ -1,107 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import weakref
import pytest
import torch
from tests.models.utils import softmax
from vllm import LLM, ClassificationRequestOutput, PoolingParams
from vllm.inputs import PromptType
from vllm.tasks import PoolingTask
MODEL_NAME = "jason9693/Qwen2.5-1.5B-apeach"
prompt = "The chef prepared a delicious meal."
prompt_token_ids = [785, 29706, 10030, 264, 17923, 15145, 13]
num_labels = 2
@pytest.fixture(scope="module")
def llm(vllm_runner):
with vllm_runner(
MODEL_NAME,
max_model_len=None,
max_num_batched_tokens=32768,
tensor_parallel_size=1,
gpu_memory_utilization=0.75,
enforce_eager=True,
seed=0,
enable_chunked_prefill=None,
) as runner:
# pytest caches yielded fixtures until after teardown, so use a proxy to
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
yield weakref.proxy(runner.llm)
@pytest.mark.skip_global_cleanup
def test_str_prompts(llm: LLM):
outputs = llm.classify(prompt, use_tqdm=False)
assert len(outputs) == 1
assert isinstance(outputs[0], ClassificationRequestOutput)
assert outputs[0].prompt_token_ids == prompt_token_ids
assert len(outputs[0].outputs.probs) == num_labels
@pytest.mark.skip_global_cleanup
def test_token_ids_prompts(llm: LLM):
outputs = llm.classify([prompt_token_ids], use_tqdm=False)
assert len(outputs) == 1
assert isinstance(outputs[0], ClassificationRequestOutput)
assert outputs[0].prompt_token_ids == prompt_token_ids
assert len(outputs[0].outputs.probs) == num_labels
@pytest.mark.skip_global_cleanup
def test_list_prompts(llm: LLM):
prompts: list[PromptType] = [prompt, prompt_token_ids]
outputs = llm.classify(prompts, use_tqdm=False)
assert len(outputs) == 2
for i in range(len(outputs)):
assert isinstance(outputs[i], ClassificationRequestOutput)
assert outputs[i].prompt_token_ids == prompt_token_ids
assert len(outputs[i].outputs.probs) == num_labels
@pytest.mark.skip_global_cleanup
def test_pooling_params(llm: LLM):
def get_outputs(use_activation):
outputs = llm.classify(
prompt,
pooling_params=PoolingParams(use_activation=use_activation),
use_tqdm=False,
)
return torch.tensor([x.outputs.probs for x in outputs])
default = get_outputs(use_activation=None)
w_activation = get_outputs(use_activation=True)
wo_activation = get_outputs(use_activation=False)
assert torch.allclose(default, w_activation, atol=1e-2), (
"Default should use activation."
)
assert not torch.allclose(w_activation, wo_activation, atol=1e-2), (
"wo_activation should not use activation."
)
assert torch.allclose(softmax(wo_activation), w_activation, atol=1e-2), (
"w_activation should be close to activation(wo_activation)."
)
@pytest.mark.skip_global_cleanup
def test_score_api(llm: LLM):
err_msg = "Scoring API is only enabled for num_labels == 1."
with pytest.raises(ValueError, match=err_msg):
llm.score("ping", "pong", use_tqdm=False)
@pytest.mark.parametrize("task", ["embed", "token_embed", "token_classify", "plugin"])
def test_unsupported_tasks(llm: LLM, task: PoolingTask):
if task == "plugin":
err_msg = "No IOProcessor plugin installed."
elif task == "token_classify":
err_msg = "Try switching the model's pooling_task via.+"
else:
err_msg = "Embedding API is not supported by this model.+"
with pytest.raises(ValueError, match=err_msg):
llm.encode(prompt, pooling_task=task, use_tqdm=False)
@@ -1,460 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import requests
import torch
import torch.nn.functional as F
from tests.utils import RemoteOpenAIServer
from vllm.entrypoints.pooling.classify.protocol import ClassificationResponse
from vllm.entrypoints.pooling.pooling.protocol import PoolingResponse
MODEL_NAME = "jason9693/Qwen2.5-1.5B-apeach"
DTYPE = "float32" # Use float32 to avoid NaN issue
input_text = "This product was excellent and exceeded my expectations"
input_tokens = [1986, 1985, 572, 9073, 323, 33808, 847, 16665]
@pytest.fixture(scope="module")
def server():
args = [
"--enforce-eager",
"--max-model-len",
"512",
"--dtype",
DTYPE,
]
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
yield remote_server
@pytest.mark.parametrize("model_name", [MODEL_NAME])
def test_basic(server: RemoteOpenAIServer, model_name: str):
# test /v1/models
response = requests.get(server.url_for("/v1/models"))
served_model = response.json()["data"][0]["id"]
assert served_model == MODEL_NAME
# test /tokenize
response = requests.post(
server.url_for("/tokenize"),
json={"model": model_name, "prompt": input_text},
)
assert response.json()["tokens"] == input_tokens
@pytest.mark.parametrize("model_name", [MODEL_NAME])
def test_completion_request(server: RemoteOpenAIServer, model_name: str):
# test input: str
classification_response = requests.post(
server.url_for("classify"),
json={"model": model_name, "input": input_text},
)
classification_response.raise_for_status()
output = ClassificationResponse.model_validate(classification_response.json())
assert output.object == "list"
assert output.model == MODEL_NAME
assert len(output.data) == 1
assert hasattr(output.data[0], "label")
assert hasattr(output.data[0], "probs")
# test input: list[int]
classification_response = requests.post(
server.url_for("classify"),
json={"model": model_name, "input": input_tokens},
)
classification_response.raise_for_status()
output = ClassificationResponse.model_validate(classification_response.json())
assert output.object == "list"
assert output.model == MODEL_NAME
assert len(output.data) == 1
assert hasattr(output.data[0], "label")
assert hasattr(output.data[0], "probs")
@pytest.mark.parametrize("model_name", [MODEL_NAME])
def test_completion_request_batched(server: RemoteOpenAIServer, model_name: str):
N = 10
# test input: list[str]
classification_response = requests.post(
server.url_for("classify"),
json={"model": model_name, "input": [input_text] * N},
)
output = ClassificationResponse.model_validate(classification_response.json())
assert len(output.data) == N
for i, item in enumerate(output.data):
assert item.index == i
assert hasattr(item, "label")
assert hasattr(item, "probs")
assert len(item.probs) == item.num_classes
assert item.label in ["Default", "Spoiled"]
# test input: list[list[int]]
classification_response = requests.post(
server.url_for("classify"),
json={"model": model_name, "input": [input_tokens] * N},
)
output = ClassificationResponse.model_validate(classification_response.json())
assert len(output.data) == N
for i, item in enumerate(output.data):
assert item.index == i
assert hasattr(item, "label")
assert hasattr(item, "probs")
assert len(item.probs) == item.num_classes
assert item.label in ["Default", "Spoiled"]
@pytest.mark.parametrize("model_name", [MODEL_NAME])
def test_empty_input_error(server: RemoteOpenAIServer, model_name: str):
classification_response = requests.post(
server.url_for("classify"),
json={"model": model_name, "input": ""},
)
error = classification_response.json()
assert classification_response.status_code == 400
assert "error" in error
classification_response = requests.post(
server.url_for("classify"),
json={"model": model_name, "input": []},
)
error = classification_response.json()
assert classification_response.status_code == 400
assert "error" in error
@pytest.mark.parametrize("model_name", [MODEL_NAME])
def test_truncate_prompt_tokens(server: RemoteOpenAIServer, model_name: str):
long_text = "hello " * 600
classification_response = requests.post(
server.url_for("classify"),
json={"model": model_name, "input": long_text, "truncate_prompt_tokens": 5},
)
classification_response.raise_for_status()
output = ClassificationResponse.model_validate(classification_response.json())
assert len(output.data) == 1
assert output.data[0].index == 0
assert hasattr(output.data[0], "probs")
assert output.usage.prompt_tokens == 5
assert output.usage.total_tokens == 5
# invalid_truncate_prompt_tokens
classification_response = requests.post(
server.url_for("classify"),
json={"model": model_name, "input": "test", "truncate_prompt_tokens": 513},
)
error = classification_response.json()
assert classification_response.status_code == 400
assert "truncate_prompt_tokens" in error["error"]["message"]
@pytest.mark.parametrize("model_name", [MODEL_NAME])
def test_add_special_tokens(server: RemoteOpenAIServer, model_name: str):
# The add_special_tokens parameter doesn't seem to be working with this model.
# working with papluca/xlm-roberta-base-language-detection
response = requests.post(
server.url_for("classify"),
json={"model": model_name, "input": input_text, "add_special_tokens": False},
)
response.raise_for_status()
ClassificationResponse.model_validate(response.json())
response = requests.post(
server.url_for("classify"),
json={"model": model_name, "input": input_text, "add_special_tokens": True},
)
response.raise_for_status()
ClassificationResponse.model_validate(response.json())
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
async def test_chat_request(server: RemoteOpenAIServer, model_name: str):
messages = [
{
"role": "user",
"content": "The cat sat on the mat.",
},
{
"role": "assistant",
"content": "A feline was resting on a rug.",
},
{
"role": "user",
"content": "Stars twinkle brightly in the night sky.",
},
]
# test chat request basic usage
response = requests.post(
server.url_for("classify"),
json={"model": model_name, "messages": messages},
)
response.raise_for_status()
output = ClassificationResponse.model_validate(response.json())
assert output.object == "list"
assert output.model == MODEL_NAME
assert len(output.data) == 1
assert hasattr(output.data[0], "label")
assert hasattr(output.data[0], "probs")
assert output.usage.prompt_tokens == 51
# test add_generation_prompt
response = requests.post(
server.url_for("classify"),
json={"model": model_name, "messages": messages, "add_generation_prompt": True},
)
response.raise_for_status()
output = ClassificationResponse.model_validate(response.json())
assert output.object == "list"
assert output.model == MODEL_NAME
assert len(output.data) == 1
assert hasattr(output.data[0], "label")
assert hasattr(output.data[0], "probs")
assert output.usage.prompt_tokens == 54
# test continue_final_message
response = requests.post(
server.url_for("classify"),
json={
"model": model_name,
"messages": messages,
"continue_final_message": True,
},
)
response.raise_for_status()
output = ClassificationResponse.model_validate(response.json())
assert output.object == "list"
assert output.model == MODEL_NAME
assert len(output.data) == 1
assert hasattr(output.data[0], "label")
assert hasattr(output.data[0], "probs")
assert output.usage.prompt_tokens == 49
# test add_special_tokens
# The add_special_tokens parameter doesn't seem to be working with this model.
response = requests.post(
server.url_for("classify"),
json={"model": model_name, "messages": messages, "add_special_tokens": True},
)
response.raise_for_status()
output = ClassificationResponse.model_validate(response.json())
assert output.object == "list"
assert output.model == MODEL_NAME
assert len(output.data) == 1
assert hasattr(output.data[0], "label")
assert hasattr(output.data[0], "probs")
assert output.usage.prompt_tokens == 51
# test continue_final_message with add_generation_prompt
response = requests.post(
server.url_for("classify"),
json={
"model": model_name,
"messages": messages,
"continue_final_message": True,
"add_generation_prompt": True,
},
)
assert (
"Cannot set both `continue_final_message` and `add_generation_prompt` to True."
in response.json()["error"]["message"]
)
@pytest.mark.asyncio
async def test_invocations_completion_request(server: RemoteOpenAIServer):
request_args = {
"model": MODEL_NAME,
"input": input_text,
}
classification_response = requests.post(
server.url_for("classify"), json=request_args
)
classification_response.raise_for_status()
invocation_response = requests.post(
server.url_for("invocations"), json=request_args
)
invocation_response.raise_for_status()
classification_output = classification_response.json()
invocation_output = invocation_response.json()
assert classification_output.keys() == invocation_output.keys()
for classification_data, invocation_data in zip(
classification_output["data"], invocation_output["data"]
):
assert classification_data.keys() == invocation_data.keys()
assert classification_data["probs"] == pytest.approx(
invocation_data["probs"], rel=0.01
)
@pytest.mark.asyncio
async def test_invocations_chat_request(server: RemoteOpenAIServer):
messages = [
{
"role": "user",
"content": "The cat sat on the mat.",
},
{
"role": "assistant",
"content": "A feline was resting on a rug.",
},
{
"role": "user",
"content": "Stars twinkle brightly in the night sky.",
},
]
request_args = {"model": MODEL_NAME, "messages": messages}
classification_response = requests.post(
server.url_for("classify"), json=request_args
)
classification_response.raise_for_status()
invocation_response = requests.post(
server.url_for("invocations"), json=request_args
)
invocation_response.raise_for_status()
classification_output = classification_response.json()
invocation_output = invocation_response.json()
assert classification_output.keys() == invocation_output.keys()
for classification_data, invocation_data in zip(
classification_output["data"], invocation_output["data"]
):
assert classification_data.keys() == invocation_data.keys()
assert classification_data["probs"] == pytest.approx(
invocation_data["probs"], rel=0.01
)
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
async def test_use_activation(server: RemoteOpenAIServer, model_name: str):
async def get_outputs(use_activation):
response = requests.post(
server.url_for("classify"),
json={
"model": model_name,
"input": input_text,
"use_activation": use_activation,
},
)
outputs = response.json()
return torch.tensor([x["probs"] for x in outputs["data"]])
default = await get_outputs(use_activation=None)
w_activation = await get_outputs(use_activation=True)
wo_activation = await get_outputs(use_activation=False)
assert torch.allclose(default, w_activation, atol=1e-2), (
"Default should use activation."
)
assert not torch.allclose(w_activation, wo_activation, atol=1e-2), (
"wo_activation should not use activation."
)
assert torch.allclose(F.softmax(wo_activation, dim=-1), w_activation, atol=1e-2), (
"w_activation should be close to activation(wo_activation)."
)
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
async def test_score(server: RemoteOpenAIServer, model_name: str):
# Scoring API is only enabled for num_labels == 1.
response = requests.post(
server.url_for("score"),
json={
"model": model_name,
"queries": "ping",
"documents": "pong",
},
)
assert response.json()["detail"] == "Not Found"
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
async def test_rerank(server: RemoteOpenAIServer, model_name: str):
# Scoring API is only enabled for num_labels == 1.
response = requests.post(
server.url_for("rerank"),
json={
"model": model_name,
"query": "ping",
"documents": ["pong"],
},
)
assert response.json()["detail"] == "Not Found"
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
async def test_pooling_classify(server: RemoteOpenAIServer, model_name: str):
response = requests.post(
server.url_for("pooling"),
json={
"model": model_name,
"input": input_text,
"encoding_format": "float",
"task": "classify",
},
)
poolings = PoolingResponse.model_validate(response.json())
assert len(poolings.data) == 1
assert len(poolings.data[0].data) == 2
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
@pytest.mark.parametrize("task", ["embed", "token_embed", "token_classify", "plugin"])
async def test_pooling_not_supported(
server: RemoteOpenAIServer, model_name: str, task: str
):
response = requests.post(
server.url_for("pooling"),
json={
"model": model_name,
"input": input_text,
"encoding_format": "float",
"task": task,
},
)
assert response.json()["error"]["type"] == "BadRequestError"
if task == "plugin":
err_msg = "No IOProcessor plugin installed."
elif task == "token_classify":
err_msg = "Try switching the model's pooling_task via"
else:
err_msg = f"Unsupported task: {task!r}"
assert response.json()["error"]["message"].startswith(err_msg)
@@ -1,147 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import pytest
import requests
from tests.utils import RemoteOpenAIServer
from vllm.assets.base import VLLM_S3_BUCKET_URL
from vllm.entrypoints.pooling.classify.protocol import ClassificationResponse
from vllm.multimodal.utils import encode_image_url, fetch_image
MODEL_NAME = "muziyongshixin/Qwen2.5-VL-7B-for-VideoCls"
MAXIMUM_VIDEOS = 1
HF_OVERRIDES = {"architectures": ["Qwen2_5_VLForSequenceClassification"]}
input_text = "This product was excellent and exceeded my expectations"
image_url = f"{VLLM_S3_BUCKET_URL}/multimodal_asset/cat_snow.jpg"
image_base64 = {"url": encode_image_url(fetch_image(image_url))}
video_url = f"{VLLM_S3_BUCKET_URL}/multimodal_asset/slow_traffic_small.mp4"
@pytest.fixture(scope="module")
def server():
args = [
"--runner",
"pooling",
"--max-model-len",
"16384",
"--enforce-eager",
"--limit-mm-per-prompt",
json.dumps({"video": MAXIMUM_VIDEOS}),
]
with RemoteOpenAIServer(
MODEL_NAME, args, override_hf_configs=HF_OVERRIDES
) as remote_server:
yield remote_server
@pytest.mark.parametrize("model_name", [MODEL_NAME])
def test_chat_text_request(server: RemoteOpenAIServer, model_name: str):
messages = [
{
"role": "assistant",
"content": "Please classify this text request.",
},
{
"role": "user",
"content": input_text,
},
]
response = requests.post(
server.url_for("classify"),
json={"model": model_name, "messages": messages},
)
response.raise_for_status()
output = ClassificationResponse.model_validate(response.json())
assert output.object == "list"
assert output.model == model_name
assert len(output.data) == 1
assert len(output.data[0].probs) == 2
assert output.usage.prompt_tokens == 35
@pytest.mark.parametrize("model_name", [MODEL_NAME])
def test_chat_image_url_request(server: RemoteOpenAIServer, model_name: str):
messages = [
{
"role": "user",
"content": [
{"type": "text", "text": "Please classify this image."},
{"type": "image_url", "image_url": {"url": image_url}},
],
}
]
response = requests.post(
server.url_for("classify"),
json={"model": model_name, "messages": messages},
)
response.raise_for_status()
output = ClassificationResponse.model_validate(response.json())
assert output.object == "list"
assert output.model == model_name
assert len(output.data) == 1
assert len(output.data[0].probs) == 2
assert output.usage.prompt_tokens == 47
@pytest.mark.parametrize("model_name", [MODEL_NAME])
def test_chat_image_base64_request(server: RemoteOpenAIServer, model_name: str):
messages = [
{
"role": "user",
"content": [
{"type": "text", "text": "Please classify this image."},
{"type": "image_url", "image_url": image_base64},
],
}
]
response = requests.post(
server.url_for("classify"),
json={"model": model_name, "messages": messages},
)
response.raise_for_status()
output = ClassificationResponse.model_validate(response.json())
assert output.object == "list"
assert output.model == model_name
assert len(output.data) == 1
assert len(output.data[0].probs) == 2
assert output.usage.prompt_tokens == 47
@pytest.mark.parametrize("model_name", [MODEL_NAME])
def test_chat_video_url_request(server: RemoteOpenAIServer, model_name: str):
messages = [
{
"role": "user",
"content": [
{"type": "text", "text": "Please classify this video."},
{"type": "video_url", "video_url": {"url": video_url}},
],
}
]
response = requests.post(
server.url_for("classify"),
json={"model": model_name, "messages": messages},
)
response.raise_for_status()
output = ClassificationResponse.model_validate(response.json())
assert output.object == "list"
assert output.model == model_name
assert len(output.data) == 1
assert len(output.data[0].probs) == 2
assert output.usage.prompt_tokens == 8993
@@ -1,310 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the Cohere /v2/embed API with generic (non-Cohere) models.
Validates that the Cohere v2 embed endpoint works correctly with standard
embedding models, covering text embedding, embedding type conversions,
response structure, batching, normalisation, and semantic similarity.
"""
import struct
import numpy as np
import pybase64 as base64
import pytest
import requests
from tests.utils import RemoteOpenAIServer
DTYPE = "bfloat16"
MODELS: list[tuple[str, list[str]]] = [
("intfloat/multilingual-e5-small", []),
(
"Snowflake/snowflake-arctic-embed-m-v1.5",
[
"--trust_remote_code",
"--hf_overrides",
'{"matryoshka_dimensions":[256]}',
],
),
]
@pytest.fixture(scope="module", params=MODELS, ids=lambda m: m[0])
def model_config(request):
return request.param
@pytest.fixture(scope="module")
def model_name(model_config):
return model_config[0]
@pytest.fixture(scope="module")
def server(model_config):
name, extra_args = model_config
args = [
"--runner",
"pooling",
"--dtype",
DTYPE,
"--enforce-eager",
"--max-model-len",
"512",
"--gpu-memory-utilization",
"0.02",
] + extra_args
with RemoteOpenAIServer(name, args) as remote_server:
yield remote_server
def _cohere_embed(
server: RemoteOpenAIServer,
model_name: str,
texts: list[str] | None = None,
images: list[str] | None = None,
input_type: str | None = None,
embedding_types: list[str] | None = None,
) -> dict:
body: dict = {"model": model_name}
if input_type is not None:
body["input_type"] = input_type
if texts is not None:
body["texts"] = texts
if images is not None:
body["images"] = images
if embedding_types is not None:
body["embedding_types"] = embedding_types
resp = requests.post(server.url_for("/v2/embed"), json=body)
resp.raise_for_status()
return resp.json()
def _openai_embed(
server: RemoteOpenAIServer, model_name: str, texts: list[str]
) -> dict:
body = {"model": model_name, "input": texts, "encoding_format": "float"}
resp = requests.post(server.url_for("/v1/embeddings"), json=body)
resp.raise_for_status()
return resp.json()
def _cosine_sim(a: list[float], b: list[float]) -> float:
va, vb = np.array(a), np.array(b)
return float(np.dot(va, vb) / (np.linalg.norm(va) * np.linalg.norm(vb)))
# -----------------------------------------------------------
# Text embedding tests
# -----------------------------------------------------------
def test_basic_embed(server: RemoteOpenAIServer, model_name: str):
r = _cohere_embed(
server, model_name, texts=["hello world"], embedding_types=["float"]
)
assert "embeddings" in r
assert len(r["embeddings"]["float"]) == 1
assert len(r["embeddings"]["float"][0]) > 0
def test_unsupported_input_type_rejected(server: RemoteOpenAIServer, model_name: str):
"""An input_type not defined in the model's prompt config should be
rejected with a 400 error."""
body = {
"model": model_name,
"input_type": "nonexistent_type",
"texts": ["hello world"],
"embedding_types": ["float"],
}
resp = requests.post(server.url_for("/v2/embed"), json=body)
assert resp.status_code == 400
assert "Unsupported input_type" in resp.json()["error"]["message"]
def test_omitted_input_type_accepted(server: RemoteOpenAIServer, model_name: str):
"""Omitting input_type should always work (no prompt prefix applied)."""
body = {
"model": model_name,
"texts": ["hello world"],
"embedding_types": ["float"],
}
resp = requests.post(server.url_for("/v2/embed"), json=body)
assert resp.status_code == 200
data = resp.json()
assert len(data["embeddings"]["float"]) == 1
def test_v1_v2_parity(server: RemoteOpenAIServer, model_name: str):
"""v1 (OpenAI) and v2 (Cohere) endpoints should produce the same
float embeddings for a generic model."""
texts = ["hello world"]
v2 = _cohere_embed(server, model_name, texts=texts, embedding_types=["float"])
v1 = _openai_embed(server, model_name, texts)
cos = _cosine_sim(v2["embeddings"]["float"][0], v1["data"][0]["embedding"])
assert cos > 0.9999, f"v1/v2 parity failed, cosine={cos}"
def test_embedding_types(server: RemoteOpenAIServer, model_name: str):
r = _cohere_embed(
server,
model_name,
texts=["test"],
embedding_types=["float", "binary", "ubinary"],
)
dim = len(r["embeddings"]["float"][0])
assert len(r["embeddings"]["binary"][0]) == dim // 8
assert len(r["embeddings"]["ubinary"][0]) == dim // 8
def test_response_structure(server: RemoteOpenAIServer, model_name: str):
r = _cohere_embed(server, model_name, texts=["test"], embedding_types=["float"])
assert "id" in r
assert "embeddings" in r
assert "texts" in r
assert r["texts"] == ["test"]
assert "meta" in r
assert r["meta"]["api_version"]["version"] == "2"
assert "billed_units" in r["meta"]
assert r["meta"]["billed_units"]["input_tokens"] > 0
assert r["meta"]["billed_units"]["image_tokens"] == 0
def test_batch(server: RemoteOpenAIServer, model_name: str):
texts = ["apple", "banana", "cherry"]
r = _cohere_embed(server, model_name, texts=texts, embedding_types=["float"])
assert len(r["embeddings"]["float"]) == 3
dim = len(r["embeddings"]["float"][0])
for emb in r["embeddings"]["float"]:
assert len(emb) == dim
def test_l2_normalized(server: RemoteOpenAIServer, model_name: str):
r = _cohere_embed(
server, model_name, texts=["hello world"], embedding_types=["float"]
)
emb = np.array(r["embeddings"]["float"][0])
assert abs(float(np.linalg.norm(emb)) - 1.0) < 0.01
def test_semantic_similarity(server: RemoteOpenAIServer, model_name: str):
r = _cohere_embed(
server,
model_name,
texts=["machine learning", "deep learning", "chocolate cake recipe"],
embedding_types=["float"],
)
embs = r["embeddings"]["float"]
cos_related = _cosine_sim(embs[0], embs[1])
cos_unrelated = _cosine_sim(embs[0], embs[2])
assert cos_related > cos_unrelated
def test_missing_input_returns_error(server: RemoteOpenAIServer, model_name: str):
body = {"model": model_name}
resp = requests.post(server.url_for("/v2/embed"), json=body)
assert resp.status_code == 400
def test_base64_embedding_type(server: RemoteOpenAIServer, model_name: str):
r = _cohere_embed(
server,
model_name,
texts=["test encoding"],
embedding_types=["float", "base64"],
)
float_emb = r["embeddings"]["float"][0]
b64_str = r["embeddings"]["base64"][0]
decoded = struct.unpack(f"<{len(float_emb)}f", base64.b64decode(b64_str))
np.testing.assert_allclose(float_emb, decoded, rtol=1e-5)
# -----------------------------------------------------------
# Truncation tests
# -----------------------------------------------------------
def _cohere_embed_raw(
server: RemoteOpenAIServer,
body: dict,
) -> requests.Response:
return requests.post(server.url_for("/v2/embed"), json=body)
def test_truncate_end_succeeds(server: RemoteOpenAIServer, model_name: str):
"""truncate=END should silently truncate long input."""
long_text = " ".join(["word"] * 2000)
body = {
"model": model_name,
"texts": [long_text],
"embedding_types": ["float"],
"truncate": "END",
}
resp = _cohere_embed_raw(server, body)
assert resp.status_code == 200
data = resp.json()
assert len(data["embeddings"]["float"]) == 1
def test_truncate_start_succeeds(server: RemoteOpenAIServer, model_name: str):
"""truncate=START should silently truncate long input from the start."""
long_text = " ".join(["word"] * 2000)
body = {
"model": model_name,
"texts": [long_text],
"embedding_types": ["float"],
"truncate": "START",
}
resp = _cohere_embed_raw(server, body)
assert resp.status_code == 200
data = resp.json()
assert len(data["embeddings"]["float"]) == 1
def test_truncate_none_rejects_long_input(server: RemoteOpenAIServer, model_name: str):
"""truncate=NONE should error when input exceeds model context."""
long_text = " ".join(["word"] * 2000)
body = {
"model": model_name,
"texts": [long_text],
"embedding_types": ["float"],
"truncate": "NONE",
}
resp = _cohere_embed_raw(server, body)
assert resp.status_code == 400
def test_truncate_start_vs_end_differ(server: RemoteOpenAIServer, model_name: str):
"""START and END truncation should produce different embeddings
when the input is long enough to actually be truncated.
We construct input with distinct tokens at the start vs end
so that keeping different halves produces different embeddings.
"""
start_words = " ".join([f"alpha{i}" for i in range(300)])
end_words = " ".join([f"omega{i}" for i in range(300)])
long_text = start_words + " " + end_words
body_end = {
"model": model_name,
"texts": [long_text],
"embedding_types": ["float"],
"truncate": "END",
}
body_start = {
"model": model_name,
"texts": [long_text],
"embedding_types": ["float"],
"truncate": "START",
}
r_end = _cohere_embed_raw(server, body_end).json()
r_start = _cohere_embed_raw(server, body_start).json()
emb_end = r_end["embeddings"]["float"][0]
emb_start = r_start["embeddings"]["float"][0]
cos = _cosine_sim(emb_end, emb_start)
assert cos < 0.99, (
f"START and END truncation should produce different embeddings "
f"for long input, but cosine similarity was {cos}"
)
@@ -1,135 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the Cohere /v2/embed API with a multimodal model (SigLIP).
Validates image embedding, batching, normalisation, and embedding type
conversions through the /v2/embed endpoint.
"""
import struct
import zlib
import numpy as np
import pybase64 as base64
import pytest
import requests
from tests.utils import RemoteOpenAIServer
MODEL_NAME = "google/siglip-so400m-patch14-384"
DTYPE = "bfloat16"
@pytest.fixture(scope="module")
def server():
args = [
"--runner",
"pooling",
"--dtype",
DTYPE,
"--enforce-eager",
"--max-model-len",
"64",
"--gpu-memory-utilization",
"0.3",
]
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
yield remote_server
def _make_tiny_png(r: int, g: int, b: int, w: int = 2, h: int = 2) -> str:
raw = b""
for _ in range(h):
raw += b"\x00" + bytes([r, g, b]) * w
compressed = zlib.compress(raw)
def chunk(ctype: bytes, cdata: bytes) -> bytes:
c = ctype + cdata
return (
struct.pack(">I", len(cdata))
+ c
+ struct.pack(">I", zlib.crc32(c) & 0xFFFFFFFF)
)
ihdr = struct.pack(">IIBBBBB", w, h, 8, 2, 0, 0, 0)
png = (
b"\x89PNG\r\n\x1a\n"
+ chunk(b"IHDR", ihdr)
+ chunk(b"IDAT", compressed)
+ chunk(b"IEND", b"")
)
return "data:image/png;base64," + base64.b64encode(png).decode()
def _cohere_embed(
server: RemoteOpenAIServer,
texts: list[str] | None = None,
images: list[str] | None = None,
embedding_types: list[str] | None = None,
) -> dict:
body: dict = {"model": MODEL_NAME}
if texts is not None:
body["texts"] = texts
if images is not None:
body["images"] = images
if embedding_types is not None:
body["embedding_types"] = embedding_types
resp = requests.post(server.url_for("/v2/embed"), json=body)
resp.raise_for_status()
return resp.json()
def test_image_embed(server: RemoteOpenAIServer):
img_uri = _make_tiny_png(255, 0, 0)
r = _cohere_embed(
server,
images=[img_uri],
embedding_types=["float"],
)
assert "embeddings" in r
assert len(r["embeddings"]["float"]) == 1
assert len(r["embeddings"]["float"][0]) > 0
assert r["meta"]["billed_units"]["image_tokens"] > 0
assert r["meta"]["billed_units"]["input_tokens"] == 0
def test_image_batch(server: RemoteOpenAIServer):
red = _make_tiny_png(255, 0, 0)
blue = _make_tiny_png(0, 0, 255)
r = _cohere_embed(
server,
images=[red, blue],
embedding_types=["float"],
)
assert len(r["embeddings"]["float"]) == 2
def test_image_l2_normalized(server: RemoteOpenAIServer):
img_uri = _make_tiny_png(0, 255, 0)
r = _cohere_embed(
server,
images=[img_uri],
embedding_types=["float"],
)
emb = np.array(r["embeddings"]["float"][0])
assert abs(float(np.linalg.norm(emb)) - 1.0) < 0.01
def test_image_embedding_types(server: RemoteOpenAIServer):
img_uri = _make_tiny_png(128, 128, 128)
r = _cohere_embed(
server,
images=[img_uri],
embedding_types=["float", "binary", "ubinary"],
)
dim = len(r["embeddings"]["float"][0])
assert len(r["embeddings"]["binary"][0]) == dim // 8
assert len(r["embeddings"]["ubinary"][0]) == dim // 8
def test_text_embed_on_multimodal(server: RemoteOpenAIServer):
"""SigLIP also supports text-only embedding via /v2/embed."""
r = _cohere_embed(server, texts=["hello world"], embedding_types=["float"])
assert "embeddings" in r
assert len(r["embeddings"]["float"]) == 1
assert len(r["embeddings"]["float"][0]) > 0
@@ -1,121 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Parity test between Cohere /v2/embed and OpenAI /v1/embeddings.
Verifies that both endpoints produce identical float embeddings when
no prompt prefix is applied (input_type omitted for Cohere /v2/embed).
"""
import numpy as np
import pytest
import requests
from tests.utils import ROCM_EXTRA_ARGS, RemoteOpenAIServer
MODEL_NAME = "BAAI/bge-base-en-v1.5"
DTYPE = "bfloat16"
@pytest.fixture(scope="module")
def server():
args = [
"--runner",
"pooling",
"--dtype",
DTYPE,
"--enforce-eager",
"--max-model-len",
"512",
"--gpu-memory-utilization",
"0.02",
] + ROCM_EXTRA_ARGS
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
yield remote_server
def _cohere_embed(
server: RemoteOpenAIServer,
texts: list[str],
) -> list[list[float]]:
body = {
"model": MODEL_NAME,
"texts": texts,
"embedding_types": ["float"],
}
resp = requests.post(server.url_for("/v2/embed"), json=body)
resp.raise_for_status()
return resp.json()["embeddings"]["float"]
def _openai_embed(
server: RemoteOpenAIServer,
texts: list[str],
) -> list[list[float]]:
body = {"model": MODEL_NAME, "input": texts, "encoding_format": "float"}
resp = requests.post(server.url_for("/v1/embeddings"), json=body)
resp.raise_for_status()
return [item["embedding"] for item in resp.json()["data"]]
def _cosine_sim(a: list[float], b: list[float]) -> float:
va, vb = np.array(a), np.array(b)
return float(np.dot(va, vb) / (np.linalg.norm(va) * np.linalg.norm(vb)))
def test_single_text_parity(server: RemoteOpenAIServer):
"""A single text should produce equivalent embeddings via both APIs."""
texts = ["the quick brown fox jumps over the lazy dog"]
v2 = _cohere_embed(server, texts)
v1 = _openai_embed(server, texts)
# Full-suite BF16 runs can introduce tiny numerical drift even when both
# endpoints are functionally equivalent, so compare semantic equivalence
# instead of exact elementwise equality.
cos = _cosine_sim(v2[0], v1[0])
assert cos > 0.9999, f"single-text parity failed, cosine={cos}"
def test_batch_parity(server: RemoteOpenAIServer):
"""A batch of texts should produce equivalent embeddings via both APIs,
in the same order."""
texts = [
"machine learning",
"deep learning",
"natural language processing",
]
v2 = _cohere_embed(server, texts)
v1 = _openai_embed(server, texts)
assert len(v2) == len(v1) == 3
similarities = np.array(
[[_cosine_sim(v2_emb, v1_emb) for v1_emb in v1] for v2_emb in v2]
)
for i in range(3):
assert int(np.argmax(similarities[i])) == i, (
f"batch parity order mismatch at index {i}: "
f"similarities={similarities[i].tolist()}"
)
assert similarities[i, i] > 0.9999, (
f"batch parity failed at index {i}, cosine={similarities[i, i]}"
)
def test_token_count_parity(server: RemoteOpenAIServer):
"""Both APIs should report the same prompt token count."""
texts = ["hello world"]
v2_resp = requests.post(
server.url_for("/v2/embed"),
json={
"model": MODEL_NAME,
"texts": texts,
"embedding_types": ["float"],
},
)
v1_resp = requests.post(
server.url_for("/v1/embeddings"),
json={"model": MODEL_NAME, "input": texts, "encoding_format": "float"},
)
v2_resp.raise_for_status()
v1_resp.raise_for_status()
v2_tokens = v2_resp.json()["meta"]["billed_units"]["input_tokens"]
v1_tokens = v1_resp.json()["usage"]["prompt_tokens"]
assert v2_tokens == v1_tokens
@@ -1,41 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
import pytest
from tests.models.language.pooling_mteb_test.mteb_embed_utils import (
MTEB_EMBED_TASKS,
MTEB_EMBED_TOL,
OpenAIClientMtebEncoder,
run_mteb_embed_task,
)
from tests.utils import RemoteOpenAIServer
os.environ["VLLM_LOGGING_LEVEL"] = "WARNING"
MODEL_NAME = "intfloat/e5-small"
MAIN_SCORE = 0.7422994752439667
@pytest.fixture(scope="module")
def server():
args = ["--runner", "pooling", "--enforce-eager", "--disable-uvicorn-access-log"]
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
yield remote_server
def test_mteb_embed(server):
client = server.get_client()
encoder = OpenAIClientMtebEncoder(MODEL_NAME, client)
vllm_main_score = run_mteb_embed_task(encoder, MTEB_EMBED_TASKS)
st_main_score = MAIN_SCORE
print("VLLM main score: ", vllm_main_score)
print("SentenceTransformer main score: ", st_main_score)
print("Difference: ", st_main_score - vllm_main_score)
# We are not concerned that the vllm mteb results are better
# than SentenceTransformers, so we only perform one-sided testing.
assert st_main_score - vllm_main_score < MTEB_EMBED_TOL
@@ -1,507 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for EmbedIOProcessor."""
import pytest
import torch
from pydantic import TypeAdapter, ValidationError
from vllm import PoolingParams
from vllm.entrypoints.pooling.embed.io_processor import EmbedIOProcessor
from vllm.entrypoints.pooling.embed.protocol import (
CohereEmbedRequest,
EmbeddingBatchChatInputRequest,
EmbeddingBatchChatRequest,
EmbeddingChatInputRequest,
EmbeddingChatRequest,
EmbeddingCompletionRequest,
EmbeddingRequest,
)
from vllm.entrypoints.pooling.typing import (
PoolingEngineInput,
PoolingServeContext,
)
from vllm.inputs import tokens_input
from vllm.outputs import PoolingOutput, PoolingRequestOutput
class TestEmbeddingRequestParsing:
"""Unit tests for OpenAI embedding request parsing."""
def test_input_messages_parses_as_chat_request(self):
request: EmbeddingRequest = TypeAdapter(EmbeddingRequest).validate_python(
{
"model": "test",
"input": [{"role": "user", "content": "hello"}],
"chat_template_kwargs": {"instruction": "Represent the query: "},
}
)
assert isinstance(request, EmbeddingChatInputRequest)
assert request.input == [{"role": "user", "content": "hello"}]
assert request.messages == [{"role": "user", "content": "hello"}]
assert request.chat_template_kwargs == {"instruction": "Represent the query: "}
def test_batched_input_messages_parses_as_batch_chat_input_request(self):
request: EmbeddingRequest = TypeAdapter(EmbeddingRequest).validate_python(
{
"model": "test",
"input": [
[{"role": "user", "content": "hello"}],
[{"role": "user", "content": "goodbye"}],
],
"chat_template_kwargs": {"instruction": "Represent the query: "},
}
)
assert isinstance(request, EmbeddingBatchChatInputRequest)
assert request.input == [
[{"role": "user", "content": "hello"}],
[{"role": "user", "content": "goodbye"}],
]
assert request.messages == [
[{"role": "user", "content": "hello"}],
[{"role": "user", "content": "goodbye"}],
]
assert request.chat_template_kwargs == {"instruction": "Represent the query: "}
def test_token_ids_still_parse_as_completion_request(self):
request: EmbeddingRequest = TypeAdapter(EmbeddingRequest).validate_python(
{
"model": "test",
"input": [[1, 2, 3], [4, 5]],
}
)
assert isinstance(request, EmbeddingCompletionRequest)
assert request.input == [[1, 2, 3], [4, 5]]
def test_messages_still_parses_as_chat_request(self):
request: EmbeddingRequest = TypeAdapter(EmbeddingRequest).validate_python(
{
"model": "test",
"messages": [{"role": "user", "content": "hello"}],
"chat_template_kwargs": {"instruction": "Represent the query: "},
}
)
assert isinstance(request, EmbeddingChatRequest)
assert request.messages == [{"role": "user", "content": "hello"}]
assert request.chat_template_kwargs == {"instruction": "Represent the query: "}
def test_batched_messages_parses_as_batch_chat_request(self):
request: EmbeddingRequest = TypeAdapter(EmbeddingRequest).validate_python(
{
"model": "test",
"messages": [
[{"role": "user", "content": "hello"}],
[{"role": "user", "content": "goodbye"}],
],
"chat_template_kwargs": {"instruction": "Represent the query: "},
}
)
assert isinstance(request, EmbeddingBatchChatRequest)
assert request.messages == [
[{"role": "user", "content": "hello"}],
[{"role": "user", "content": "goodbye"}],
]
assert request.chat_template_kwargs == {"instruction": "Represent the query: "}
class TestCohereEmbedRequestParsing:
"""Unit tests for Cohere embed request parsing."""
@pytest.mark.parametrize(
"request_body",
[
{"model": "test"},
{"model": "test", "texts": ["hello"], "images": ["image-uri"]},
{
"model": "test",
"texts": ["hello"],
"inputs": [
{"content": [{"type": "text", "text": "hello"}]},
],
},
{
"model": "test",
"images": ["image-uri"],
"inputs": [
{"content": [{"type": "text", "text": "hello"}]},
],
},
{"model": "test", "texts": []},
{"model": "test", "images": []},
{"model": "test", "inputs": []},
],
)
def test_rejects_invalid_input_field_combinations(self, request_body):
with pytest.raises(
ValidationError,
match="Exactly one of texts, images, or inputs must be provided",
):
CohereEmbedRequest(**request_body)
@pytest.mark.parametrize(
"request_body",
[
{"model": "test", "texts": ["hello"]},
{"model": "test", "images": ["image-uri"]},
{
"model": "test",
"inputs": [
{"content": [{"type": "text", "text": "hello"}]},
],
},
{
"model": "test",
"inputs": [
{
"content": [
{"type": "image_url", "image_url": {"url": "image-uri"}}
]
},
],
},
],
)
def test_accepts_exactly_one_non_empty_input_field(self, request_body):
request = CohereEmbedRequest(**request_body)
assert request.model == "test"
@pytest.mark.parametrize(
("content", "error"),
[
(
{"type": "text"},
"CohereEmbedContent with type='text' requires text",
),
(
{"type": "image_url"},
"CohereEmbedContent with type='image_url' requires image_url.url",
),
(
{"type": "image_url", "image_url": {}},
"CohereEmbedContent with type='image_url' requires image_url.url",
),
(
{"type": "image_url", "image_url": {"url": ""}},
"CohereEmbedContent with type='image_url' requires image_url.url",
),
],
)
def test_rejects_invalid_mixed_content_payloads(self, content, error):
with pytest.raises(ValidationError, match=error):
CohereEmbedRequest(
model="test",
inputs=[
{
"content": [content],
},
],
)
class TestResolveTruncation:
"""Unit tests for EmbedIOProcessor._resolve_cohere_truncation."""
@staticmethod
def _make_request(**kwargs) -> CohereEmbedRequest:
defaults = {
"model": "test",
"input_type": "search_document",
"texts": ["hello"],
}
return CohereEmbedRequest(**(defaults | kwargs))
def test_truncate_end_default(self):
req = self._make_request()
tokens, side = EmbedIOProcessor._resolve_cohere_truncation(req)
assert tokens == -1
assert side is None
def test_truncate_end_explicit(self):
req = self._make_request(truncate="END")
tokens, side = EmbedIOProcessor._resolve_cohere_truncation(req)
assert tokens == -1
assert side is None
def test_truncate_end_with_max_tokens(self):
req = self._make_request(truncate="END", max_tokens=128)
tokens, side = EmbedIOProcessor._resolve_cohere_truncation(req)
assert tokens == 128
assert side is None
def test_truncate_none(self):
req = self._make_request(truncate="NONE")
tokens, side = EmbedIOProcessor._resolve_cohere_truncation(req)
assert tokens is None
assert side is None
def test_truncate_none_with_max_tokens(self):
"""truncate=NONE should NOT set truncate_prompt_tokens; the
max_tokens limit is enforced separately via _check_max_tokens."""
req = self._make_request(truncate="NONE", max_tokens=10)
tokens, side = EmbedIOProcessor._resolve_cohere_truncation(req)
assert tokens is None
assert side is None
def test_truncate_start(self):
req = self._make_request(truncate="START")
tokens, side = EmbedIOProcessor._resolve_cohere_truncation(req)
assert tokens == -1
assert side == "left"
def test_truncate_start_with_max_tokens(self):
req = self._make_request(truncate="START", max_tokens=64)
tokens, side = EmbedIOProcessor._resolve_cohere_truncation(req)
assert tokens == 64
assert side == "left"
class TestApplyStPrompt:
"""Unit tests for EmbedIOProcessor._apply_task_instruction."""
@staticmethod
def _make_handler(task_instructions: dict[str, str] | None):
handler = object.__new__(EmbedIOProcessor)
handler.task_instructions = task_instructions
return handler
def test_no_prompts_configured(self):
handler = self._make_handler(None)
texts = ["hello", "world"]
assert handler._apply_task_instruction(texts, "query") is texts
def test_matching_input_type(self):
handler = self._make_handler({"query": "search_query: "})
result = handler._apply_task_instruction(["hello"], "query")
assert result == ["search_query: hello"]
def test_non_matching_input_type(self):
handler = self._make_handler({"query": "search_query: "})
texts = ["hello"]
assert handler._apply_task_instruction(texts, "document") is texts
def test_multiple_texts(self):
handler = self._make_handler(
{"query": "Represent this sentence for searching: "}
)
result = handler._apply_task_instruction(["a", "b", "c"], "query")
assert result == [
"Represent this sentence for searching: a",
"Represent this sentence for searching: b",
"Represent this sentence for searching: c",
]
def test_empty_prefix_returns_unchanged(self):
handler = self._make_handler({"passage": ""})
texts = ["hello"]
assert handler._apply_task_instruction(texts, "passage") is texts
class TestLoadTaskInstructions:
"""Unit tests for EmbedIOProcessor._load_task_instructions."""
def test_no_attribute(self):
class FakeConfig:
pass
assert EmbedIOProcessor._load_task_instructions(FakeConfig()) is None
def test_with_task_instructions(self):
class FakeConfig:
task_instructions = {
"retrieval.query": "Represent the query: ",
"retrieval.passage": "",
}
result = EmbedIOProcessor._load_task_instructions(FakeConfig())
assert result == {
"retrieval.query": "Represent the query: ",
"retrieval.passage": "",
}
def test_empty_dict(self):
class FakeConfig:
task_instructions = {}
assert EmbedIOProcessor._load_task_instructions(FakeConfig()) is None
def test_non_dict(self):
class FakeConfig:
task_instructions = "not a dict"
assert EmbedIOProcessor._load_task_instructions(FakeConfig()) is None
class TestCheckMaxTokens:
"""Unit tests for EmbedIOProcessor._check_cohere_max_tokens."""
@staticmethod
def _fake_output(n_tokens: int):
class _Out:
def __init__(self, n: int):
self.prompt_token_ids = list(range(n))
return _Out(n_tokens)
def test_none_check_is_noop(self):
outs = [self._fake_output(100)]
EmbedIOProcessor._check_cohere_max_tokens(outs, None)
def test_within_limit(self):
outs = [self._fake_output(5), self._fake_output(3)]
EmbedIOProcessor._check_cohere_max_tokens(outs, 5)
def test_exceeds_limit(self):
outs = [self._fake_output(3), self._fake_output(10)]
with pytest.raises(ValueError, match="exceeds max_tokens=5"):
EmbedIOProcessor._check_cohere_max_tokens(outs, 5)
def test_exact_limit(self):
outs = [self._fake_output(5)]
EmbedIOProcessor._check_cohere_max_tokens(outs, 5)
class TestValidateInputType:
"""Unit tests for EmbedIOProcessor._validate_input_type."""
@staticmethod
def _make_handler(task_instructions: dict[str, str] | None):
handler = object.__new__(EmbedIOProcessor)
handler.task_instructions = task_instructions
return handler
def test_none_input_type_always_accepted(self):
handler = self._make_handler(None)
handler._validate_input_type(None)
handler_with = self._make_handler({"query": "q: "})
handler_with._validate_input_type(None)
def test_no_prompts_rejects(self):
handler = self._make_handler(None)
with pytest.raises(ValueError, match="does not define any input_type"):
handler._validate_input_type("anything")
def test_known_type_accepted(self):
handler = self._make_handler({"query": "q: ", "document": "d: "})
handler._validate_input_type("query")
handler._validate_input_type("document")
def test_unknown_type_rejected(self):
handler = self._make_handler({"query": "q: ", "document": "d: "})
with pytest.raises(ValueError, match="Unsupported input_type 'other'"):
handler._validate_input_type("other")
def test_error_lists_supported(self):
handler = self._make_handler({"a": "", "b": ""})
with pytest.raises(ValueError, match="Supported values: a, b"):
handler._validate_input_type("z")
class TestChunkedEmbeddingProcessing:
"""Unit tests for chunked embedding aggregation."""
class _FakeModelConfig:
max_model_len = 3
@classmethod
def _make_handler(cls):
handler = object.__new__(EmbedIOProcessor)
handler.model_config = cls._FakeModelConfig()
handler.enable_chunked_processing = True
return handler
@staticmethod
def _make_context() -> PoolingServeContext[EmbeddingCompletionRequest]:
request: EmbeddingRequest = TypeAdapter(EmbeddingRequest).validate_python(
{
"model": "test",
"input": [[0, 1, 2, 3, 4], [10, 11]],
}
)
assert isinstance(request, EmbeddingCompletionRequest)
pooling_params = PoolingParams()
return PoolingServeContext(
request=request,
pooling_params=pooling_params,
model_name="test",
request_id="embd-client-prompt-999-chunk-888",
engine_inputs=[
PoolingEngineInput(
prompts=tokens_input(prompt_token_ids=[0, 1, 2, 3, 4]),
params=pooling_params,
lora_requests=None,
priorities=0,
),
PoolingEngineInput(
prompts=tokens_input(prompt_token_ids=[10, 11]),
params=pooling_params,
lora_requests=None,
priorities=0,
),
],
lora_request=None,
priorities=0,
prompt_extras=None,
)
@staticmethod
def _make_output(
request_id: str,
prompt_token_ids: list[int],
embedding: list[float],
) -> PoolingRequestOutput:
return PoolingRequestOutput(
request_id=request_id,
outputs=PoolingOutput(data=torch.tensor(embedding)),
prompt_token_ids=prompt_token_ids,
num_cached_tokens=0,
finished=True,
)
def test_aggregation_uses_metadata_not_request_id_parsing(self):
handler = self._make_handler()
ctx = self._make_context()
handler.maybe_pre_process_chunked(ctx)
assert ctx.prompt_request_ids == [
"embd-client-prompt-999-chunk-888-prompt-0-chunk-0",
"embd-client-prompt-999-chunk-888-prompt-0-chunk-1",
"embd-client-prompt-999-chunk-888-prompt-1-chunk-0",
]
assert ctx.chunked_embedding_metadata is not None
assert [
(item.prompt_index, item.chunk_index)
for item in ctx.chunked_embedding_metadata
] == [(0, 0), (0, 1), (1, 0)]
ctx.final_res_batch = [
self._make_output(ctx.prompt_request_ids[0], [0, 1, 2], [1.0, 1.0]),
self._make_output(ctx.prompt_request_ids[1], [3, 4], [4.0, 7.0]),
self._make_output(ctx.prompt_request_ids[2], [10, 11], [9.0, 9.0]),
]
handler._post_process_chunked(ctx)
assert len(ctx.final_res_batch) == 2
assert ctx.final_res_batch[0].request_id == (
"embd-client-prompt-999-chunk-888-prompt-0"
)
assert ctx.final_res_batch[0].prompt_token_ids == [0, 1, 2, 3, 4]
assert torch.allclose(
ctx.final_res_batch[0].outputs.data,
torch.tensor([2.2, 3.4]),
)
assert ctx.final_res_batch[1].request_id == (
"embd-client-prompt-999-chunk-888-prompt-1"
)
assert ctx.final_res_batch[1].prompt_token_ids == [10, 11]
assert torch.allclose(
ctx.final_res_batch[1].outputs.data,
torch.tensor([9.0, 9.0]),
)
@@ -1,101 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import weakref
import pytest
import torch
import torch.nn.functional as F
from vllm import LLM, EmbeddingRequestOutput, PoolingParams
from vllm.inputs import PromptType
from vllm.tasks import PoolingTask
MODEL_NAME = "intfloat/multilingual-e5-small"
prompt = "The chef prepared a delicious meal."
prompt_token_ids = [0, 581, 21861, 133888, 10, 8, 150, 60744, 109911, 5, 2]
embedding_size = 384
@pytest.fixture(scope="module")
def llm(vllm_runner):
with vllm_runner(
MODEL_NAME,
max_model_len=None,
max_num_batched_tokens=32768,
tensor_parallel_size=1,
gpu_memory_utilization=0.75,
enforce_eager=True,
seed=0,
enable_chunked_prefill=None,
) as runner:
assert embedding_size == runner.llm.model_config.embedding_size
# pytest caches yielded fixtures until after teardown, so use a proxy to
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
yield weakref.proxy(runner.llm)
@pytest.mark.skip_global_cleanup
def test_str_prompts(llm: LLM):
outputs = llm.embed(prompt, use_tqdm=False)
assert len(outputs) == 1
assert isinstance(outputs[0], EmbeddingRequestOutput)
assert outputs[0].prompt_token_ids == prompt_token_ids
assert len(outputs[0].outputs.embedding) == embedding_size
@pytest.mark.skip_global_cleanup
def test_token_ids_prompts(llm: LLM):
outputs = llm.embed([prompt_token_ids], use_tqdm=False)
assert len(outputs) == 1
assert isinstance(outputs[0], EmbeddingRequestOutput)
assert outputs[0].prompt_token_ids == prompt_token_ids
assert len(outputs[0].outputs.embedding) == embedding_size
@pytest.mark.skip_global_cleanup
def test_list_prompts(llm: LLM):
prompts: list[PromptType] = [prompt, prompt_token_ids]
outputs = llm.embed(prompts, use_tqdm=False)
assert len(outputs) == 2
for i in range(len(outputs)):
assert isinstance(outputs[i], EmbeddingRequestOutput)
assert outputs[i].prompt_token_ids == prompt_token_ids
assert len(outputs[i].outputs.embedding) == embedding_size
@pytest.mark.skip_global_cleanup
def test_pooling_params(llm: LLM):
def get_outputs(normalize):
outputs = llm.embed(
[prompt],
pooling_params=PoolingParams(use_activation=normalize),
use_tqdm=False,
)
return torch.tensor([x.outputs.embedding for x in outputs])
default = get_outputs(normalize=None)
w_normal = get_outputs(normalize=True)
wo_normal = get_outputs(normalize=False)
assert torch.allclose(default, w_normal, atol=1e-2), "Default should use normal."
assert not torch.allclose(w_normal, wo_normal, atol=1e-2), (
"wo_normal should not use normal."
)
assert torch.allclose(w_normal, F.normalize(wo_normal, p=2, dim=-1), atol=1e-2), (
"w_normal should be close to normal(wo_normal)."
)
@pytest.mark.parametrize(
"task", ["token_classify", "classify", "token_embed", "plugin"]
)
def test_unsupported_tasks(llm: LLM, task: PoolingTask):
if task == "plugin":
err_msg = "No IOProcessor plugin installed."
elif task == "token_embed":
err_msg = "Try switching the model's pooling_task via.+"
else:
err_msg = "Classification API is not supported by this model.+"
with pytest.raises(ValueError, match=err_msg):
llm.encode(prompt, pooling_task=task, use_tqdm=False)
@@ -1,801 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import numpy as np
import openai
import pybase64 as base64
import pytest
import pytest_asyncio
import requests
import torch
import torch.nn.functional as F
from tests.models.language.pooling.embed_utils import (
float_embeddings,
run_embedding_correctness_test,
)
from tests.models.utils import check_embeddings_close
from tests.utils import RemoteOpenAIServer
from vllm.entrypoints.pooling.embed.protocol import EmbeddingResponse
from vllm.entrypoints.pooling.pooling.protocol import PoolingResponse
from vllm.entrypoints.pooling.utils import (
MetadataItem,
build_metadata_items,
decode_pooling_output,
)
from vllm.platforms import current_platform
from vllm.renderers.chat_utils import ChatCompletionMessageParam
from vllm.tokenizers import get_tokenizer
from vllm.utils.serial_utils import EMBED_DTYPES, ENDIANNESS, binary2tensor
MODEL_NAME = "intfloat/multilingual-e5-small"
DUMMY_CHAT_TEMPLATE = """{% for message in messages %}{{message['role'] + ': ' + message['content'] + '\\n'}}{% endfor %}""" # noqa: E501
DTYPE = "bfloat16"
input_text = "The best thing about vLLM is that it supports many different models"
input_tokens = [
0,
581,
2965,
13580,
1672,
81,
23708,
594,
83,
450,
442,
8060,
7,
5941,
12921,
115774,
2,
]
# On ROCm, floating-point reductions in attention and GEMM kernels are
# non-associative and sensitive to batch geometry. Force LLM instances
# into an identical, deterministic execution mode:
ROCM_DETERMINISM_ARGS: list[str] = (
["--max-num-seqs", "1"] if current_platform.is_rocm() else []
)
@pytest.fixture(scope="module")
def server():
args = [
"--runner",
"pooling",
"--dtype",
DTYPE,
"--enforce-eager",
"--max-model-len",
"512",
"--chat-template",
DUMMY_CHAT_TEMPLATE,
*ROCM_DETERMINISM_ARGS,
]
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
yield remote_server
@pytest_asyncio.fixture
async def client(server):
async with server.get_async_client() as async_client:
yield async_client
@pytest.fixture(scope="module")
def hf_model(hf_runner):
with hf_runner(MODEL_NAME, dtype=DTYPE, is_sentence_transformer=True) as hf_model:
yield hf_model
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
async def test_negative_token_ids(client: openai.AsyncOpenAI, model_name: str):
# A negative token id is out of vocabulary just like an over-large one, but
# is not caught by the upper-bound check. Unlike the OpenAI completion
# schema, the pooling schema does not constrain token ids to be
# non-negative, so the request reaches the shared engine-level validation.
with pytest.raises(openai.BadRequestError, match=".*out of vocabulary.*"):
await client.embeddings.create(model=model_name, input=[-1])
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
async def test_basic(
server: RemoteOpenAIServer, client: openai.AsyncOpenAI, model_name: str
):
# test /v1/models
response = requests.get(server.url_for("/v1/models"))
model = response.json()["data"][0]["id"]
assert model == MODEL_NAME
models = await client.models.list()
models = models.data
served_model = models[0]
assert served_model.id == MODEL_NAME
# test /tokenize
response = requests.post(
server.url_for("/tokenize"),
json={"model": model_name, "prompt": input_text},
)
assert response.json()["tokens"] == input_tokens
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
async def test_completion_request(
client: openai.AsyncOpenAI, model_name: str, hf_model
):
# test input: str
embedding_response = await client.embeddings.create(
model=model_name,
input=input_text,
encoding_format="float",
)
embeddings = EmbeddingResponse.model_validate(
embedding_response.model_dump(mode="json")
)
assert embeddings.id is not None
assert len(embeddings.data) == 1
assert len(embeddings.data[0].embedding) == 384
assert embeddings.usage.completion_tokens == 0
assert embeddings.usage.prompt_tokens == len(input_tokens)
assert embeddings.usage.total_tokens == len(input_tokens)
vllm_outputs = float_embeddings(embeddings)
run_embedding_correctness_test(hf_model, [input_text], vllm_outputs)
# test input: list[int]
embedding_response = await client.embeddings.create(
model=model_name,
input=input_tokens,
encoding_format="float",
)
embeddings = EmbeddingResponse.model_validate(
embedding_response.model_dump(mode="json")
)
assert embeddings.id is not None
assert len(embeddings.data) == 1
assert len(embeddings.data[0].embedding) == 384
assert embeddings.usage.completion_tokens == 0
assert embeddings.usage.prompt_tokens == len(input_tokens)
assert embeddings.usage.total_tokens == len(input_tokens)
vllm_outputs = float_embeddings(embeddings)
run_embedding_correctness_test(hf_model, [input_text], vllm_outputs)
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
async def test_completion_request_batched(
client: openai.AsyncOpenAI, model_name: str, hf_model
):
N = 10
input_texts = [input_text] * N
# test input: list[str]
embedding_response = await client.embeddings.create(
model=model_name,
input=input_texts,
encoding_format="float",
)
embeddings = EmbeddingResponse.model_validate(
embedding_response.model_dump(mode="json")
)
assert embeddings.id is not None
assert len(embeddings.data) == N
assert len(embeddings.data[0].embedding) == 384
assert embeddings.usage.completion_tokens == 0
assert embeddings.usage.prompt_tokens == len(input_tokens) * N
assert embeddings.usage.total_tokens == len(input_tokens) * N
vllm_outputs = float_embeddings(embeddings)
run_embedding_correctness_test(hf_model, input_texts, vllm_outputs)
# test list[list[int]]
embedding_response = await client.embeddings.create(
model=model_name,
input=[input_tokens] * N,
encoding_format="float",
)
embeddings = EmbeddingResponse.model_validate(
embedding_response.model_dump(mode="json")
)
assert embeddings.id is not None
assert len(embeddings.data) == N
assert len(embeddings.data[0].embedding) == 384
assert embeddings.usage.completion_tokens == 0
assert embeddings.usage.prompt_tokens == len(input_tokens) * N
assert embeddings.usage.total_tokens == len(input_tokens) * N
vllm_outputs = float_embeddings(embeddings)
run_embedding_correctness_test(hf_model, input_texts, vllm_outputs)
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
async def test_truncate_prompt_tokens(client: openai.AsyncOpenAI, model_name: str):
input_texts = [
"Como o Brasil pode fomentar o desenvolvimento de modelos de IA?",
]
# test single embedding
embedding_response = await client.embeddings.create(
model=model_name, input=input_texts, extra_body={"truncate_prompt_tokens": 10}
)
embeddings = EmbeddingResponse.model_validate(
embedding_response.model_dump(mode="json")
)
assert embeddings.id is not None
assert len(embeddings.data) == 1
assert len(embeddings.data[0].embedding) == 384
assert embeddings.usage.completion_tokens == 0
assert embeddings.usage.prompt_tokens == 10
assert embeddings.usage.total_tokens == 10
input_tokens = [
1,
24428,
289,
18341,
26165,
285,
19323,
283,
289,
26789,
3871,
28728,
9901,
340,
2229,
385,
340,
315,
28741,
28804,
2,
]
embedding_response = await client.embeddings.create(
model=model_name, input=input_tokens, extra_body={"truncate_prompt_tokens": 10}
)
embeddings = EmbeddingResponse.model_validate(
embedding_response.model_dump(mode="json")
)
assert embeddings.id is not None
assert len(embeddings.data) == 1
assert len(embeddings.data[0].embedding) == 384
assert embeddings.usage.completion_tokens == 0
assert embeddings.usage.prompt_tokens == 10
assert embeddings.usage.total_tokens == 10
# invalid_truncate_prompt_tokens
input_texts = [
"Como o Brasil pode fomentar o desenvolvimento de modelos de IA?",
]
with pytest.raises(openai.BadRequestError):
response = await client.embeddings.create(
model=model_name,
input=input_texts,
extra_body={"truncate_prompt_tokens": 8193},
)
assert "error" in response.object
assert (
"truncate_prompt_tokens value is greater than max_model_len. "
"Please request a smaller truncation size." in response.message
)
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
async def test_padding(client: openai.AsyncOpenAI, model_name: str):
input_texts = ["Hello"]
async def prompt_tokens(**extra_body):
response = await client.embeddings.create(
model=model_name,
input=input_texts,
extra_body=extra_body,
)
embeddings = EmbeddingResponse.model_validate(response.model_dump(mode="json"))
assert len(embeddings.data) == 1
return embeddings.usage.prompt_tokens
unpadded = await prompt_tokens()
padded = await prompt_tokens(padding="max_length")
# Models trained with a fixed sequence length and no attention mask need
# this; without it their embeddings are not comparable.
assert padded > unpadded
assert await prompt_tokens(padding="do_not_pad") == unpadded
with pytest.raises(openai.BadRequestError):
await client.embeddings.create(
model=model_name,
input=input_texts,
extra_body={"padding": "longest"},
)
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
async def test_chat_request(
server: RemoteOpenAIServer, client: openai.AsyncOpenAI, model_name: str
):
messages: list[ChatCompletionMessageParam] = [
{
"role": "user",
"content": "The cat sat on the mat.",
},
{
"role": "assistant",
"content": "A feline was resting on a rug.",
},
{
"role": "user",
"content": "Stars twinkle brightly in the night sky.",
},
]
# test chat request basic usage
chat_response = requests.post(
server.url_for("v1/embeddings"),
json={
"model": model_name,
"messages": messages,
"encoding_format": "float",
},
)
chat_response.raise_for_status()
chat_embeddings = EmbeddingResponse.model_validate(chat_response.json())
tokenizer = get_tokenizer(tokenizer_name=model_name)
prompt = tokenizer.apply_chat_template(
messages,
chat_template=DUMMY_CHAT_TEMPLATE,
add_generation_prompt=True,
continue_final_message=False,
tokenize=False,
)
completion_response = await client.embeddings.create(
model=model_name,
input=prompt,
encoding_format="float",
# To be consistent with chat
extra_body={"add_special_tokens": False},
)
completion_embeddings = EmbeddingResponse.model_validate(
completion_response.model_dump(mode="json")
)
assert chat_embeddings.id is not None
assert completion_embeddings.id is not None
assert chat_embeddings.created <= completion_embeddings.created
# Use tolerance-based comparison for embeddings
check_embeddings_close(
embeddings_0_lst=float_embeddings(chat_embeddings),
embeddings_1_lst=float_embeddings(completion_embeddings),
name_0="chat",
name_1="completion",
)
assert chat_embeddings.model_dump(exclude={"id", "created", "data"}) == (
completion_embeddings.model_dump(exclude={"id", "created", "data"})
)
# test add_generation_prompt
response = requests.post(
server.url_for("v1/embeddings"),
json={"model": model_name, "messages": messages, "add_generation_prompt": True},
)
response.raise_for_status()
output = EmbeddingResponse.model_validate(response.json())
assert output.object == "list"
assert len(output.data) == 1
assert output.model == MODEL_NAME
assert output.usage.prompt_tokens == 33
# test continue_final_message
response = requests.post(
server.url_for("v1/embeddings"),
json={
"model": model_name,
"messages": messages,
"continue_final_message": True,
},
)
response.raise_for_status()
output = EmbeddingResponse.model_validate(response.json())
assert output.object == "list"
assert len(output.data) == 1
assert output.model == MODEL_NAME
assert output.usage.prompt_tokens == 33
# test add_special_tokens
response = requests.post(
server.url_for("v1/embeddings"),
json={"model": model_name, "messages": messages, "add_special_tokens": True},
)
response.raise_for_status()
output = EmbeddingResponse.model_validate(response.json())
assert output.object == "list"
assert len(output.data) == 1
assert output.model == MODEL_NAME
assert output.usage.prompt_tokens == 35
# test continue_final_message with add_generation_prompt
response = requests.post(
server.url_for("v1/embeddings"),
json={
"model": model_name,
"messages": messages,
"continue_final_message": True,
"add_generation_prompt": True,
},
)
assert (
"Cannot set both `continue_final_message` and `add_generation_prompt` to True."
in response.json()["error"]["message"]
)
@pytest.mark.asyncio
async def test_invocations_completion_request(
server: RemoteOpenAIServer, client: openai.AsyncOpenAI
):
request_args = {
"model": MODEL_NAME,
"input": input_text,
"encoding_format": "float",
}
completion_response = await client.embeddings.create(**request_args)
invocation_response = requests.post(
server.url_for("invocations"), json=request_args
)
invocation_response.raise_for_status()
completion_output = completion_response.model_dump()
invocation_output = invocation_response.json()
assert completion_output.keys() == invocation_output.keys()
for completion_data, invocation_data in zip(
completion_output["data"], invocation_output["data"]
):
assert completion_data.keys() == invocation_data.keys()
check_embeddings_close(
embeddings_0_lst=[completion_data["embedding"]],
embeddings_1_lst=[invocation_data["embedding"]],
name_0="completion",
name_1="invocation",
)
@pytest.mark.asyncio
async def test_invocations_chat_request(server: RemoteOpenAIServer):
messages: list[ChatCompletionMessageParam] = [
{
"role": "user",
"content": "The cat sat on the mat.",
},
{
"role": "assistant",
"content": "A feline was resting on a rug.",
},
{
"role": "user",
"content": "Stars twinkle brightly in the night sky.",
},
]
request_args = {
"model": MODEL_NAME,
"messages": messages,
"encoding_format": "float",
}
chat_response = requests.post(server.url_for("v1/embeddings"), json=request_args)
chat_response.raise_for_status()
invocation_response = requests.post(
server.url_for("invocations"), json=request_args
)
invocation_response.raise_for_status()
chat_output = chat_response.json()
invocation_output = invocation_response.json()
assert chat_output.keys() == invocation_output.keys()
for chat_data, invocation_data in zip(
chat_output["data"], invocation_output["data"]
):
assert chat_data.keys() == invocation_data.keys()
check_embeddings_close(
embeddings_0_lst=[chat_data["embedding"]],
embeddings_1_lst=[invocation_data["embedding"]],
name_0="chat",
name_1="invocation",
)
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
async def test_base64_embedding(hf_model, client: openai.AsyncOpenAI, model_name: str):
input_texts = [
"Hello my name is",
"The best thing about vLLM is that it supports many different models",
]
def check_outputs(outputs: list[list[float]]) -> None:
# The embeddings endpoint executes list elements as separate scheduler
# requests, so use the same per-request geometry for the HF reference.
for input_text, output in zip(input_texts, outputs, strict=True):
run_embedding_correctness_test(hf_model, [input_text], [output])
responses_float = await client.embeddings.create(
input=input_texts, model=model_name, encoding_format="float"
)
float_data = [d.embedding for d in responses_float.data]
check_outputs(float_data)
responses_base64 = await client.embeddings.create(
input=input_texts, model=model_name, encoding_format="base64"
)
base64_data = []
for data in responses_base64.data:
base64_data.append(
np.frombuffer(base64.b64decode(data.embedding), dtype="float32").tolist()
)
check_outputs(base64_data)
# Default response is float32 decoded from base64 by OpenAI Client
responses_default = await client.embeddings.create(
input=input_texts, model=model_name
)
default_data = [d.embedding for d in responses_default.data]
check_outputs(default_data)
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
async def test_base64_embed_dtype_and_endianness(
server: RemoteOpenAIServer, client: openai.AsyncOpenAI, model_name: str
):
input_texts = [input_text] * 3
responses_float = await client.embeddings.create(
input=input_texts, model=model_name, encoding_format="float"
)
float_data = [d.embedding for d in responses_float.data]
for embed_dtype in EMBED_DTYPES:
for endianness in ENDIANNESS:
responses_base64 = requests.post(
server.url_for("/v1/embeddings"),
json={
"model": model_name,
"input": input_texts,
"encoding_format": "base64",
"embed_dtype": embed_dtype,
"endianness": endianness,
},
)
base64_data = []
for data in responses_base64.json()["data"]:
binary = base64.b64decode(data["embedding"])
tensor = binary2tensor(binary, (-1,), embed_dtype, endianness)
base64_data.append(tensor.to(torch.float32).tolist())
check_embeddings_close(
embeddings_0_lst=float_data,
embeddings_1_lst=base64_data,
name_0="float_data",
name_1="base64_data",
tol=1e-2,
)
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
async def test_bytes_embed_dtype_and_endianness(
server: RemoteOpenAIServer, client: openai.AsyncOpenAI, model_name: str
):
input_texts = [input_text] * 3
responses_float = await client.embeddings.create(
input=input_texts, model=model_name, encoding_format="float"
)
float_data = [d.embedding for d in responses_float.data]
for embed_dtype in EMBED_DTYPES:
for endianness in ENDIANNESS:
responses_bytes = requests.post(
server.url_for("/v1/embeddings"),
json={
"model": model_name,
"input": input_texts,
"encoding_format": "bytes",
"embed_dtype": embed_dtype,
"endianness": endianness,
},
)
metadata = json.loads(responses_bytes.headers["metadata"])
body = responses_bytes.content
items = [MetadataItem(**x) for x in metadata["data"]]
bytes_data = decode_pooling_output(items=items, body=body)
bytes_data = [x.to(torch.float32).tolist() for x in bytes_data]
check_embeddings_close(
embeddings_0_lst=float_data,
embeddings_1_lst=bytes_data,
name_0="float_data",
name_1="bytes_data",
tol=1e-2,
)
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
async def test_bytes_only_embed_dtype_and_endianness(
server: RemoteOpenAIServer, client: openai.AsyncOpenAI, model_name: str
):
input_texts = [
"The best thing about vLLM is that it supports many different models",
] * 2
responses_float = await client.embeddings.create(
input=input_texts, model=model_name, encoding_format="float"
)
float_data = [d.embedding for d in responses_float.data]
embedding_size = len(float_data[0])
for embed_dtype in EMBED_DTYPES:
for endianness in ENDIANNESS:
responses_bytes = requests.post(
server.url_for("/v1/embeddings"),
json={
"model": model_name,
"input": input_texts,
"encoding_format": "bytes_only",
"embed_dtype": embed_dtype,
"endianness": endianness,
},
)
assert "metadata" not in responses_bytes.headers
body = responses_bytes.content
items = build_metadata_items(
embed_dtype=embed_dtype,
endianness=endianness,
shape=(embedding_size,),
n_request=len(input_texts),
)
bytes_data = decode_pooling_output(items=items, body=body)
bytes_data = [x.to(torch.float32).tolist() for x in bytes_data]
check_embeddings_close(
embeddings_0_lst=float_data,
embeddings_1_lst=bytes_data,
name_0="float_data",
name_1="bytes_data",
tol=1e-2,
)
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
@pytest.mark.parametrize("param_name", ["encoding_format", "embed_dtype", "endianness"])
async def test_params_not_supported(
server: RemoteOpenAIServer, model_name: str, param_name: str
):
responses_base64 = requests.post(
server.url_for("/v1/embeddings"),
json={
"model": model_name,
"input": input_text,
"encoding_format": "base64",
param_name: f"bad_{param_name}",
},
)
assert responses_base64.status_code == 400
assert "literal_error" in responses_base64.json()["error"]["message"]
assert f"bad_{param_name}" in responses_base64.json()["error"]["message"]
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
async def test_use_activation(server: RemoteOpenAIServer, model_name: str):
async def get_outputs(use_activation):
request_args = {
"model": MODEL_NAME,
"input": input_text,
"encoding_format": "float",
"use_activation": use_activation,
}
response = requests.post(server.url_for("v1/embeddings"), json=request_args)
outputs = response.json()
return torch.tensor([x["embedding"] for x in outputs["data"]])
default = await get_outputs(use_activation=None)
w_normal = await get_outputs(use_activation=True)
wo_normal = await get_outputs(use_activation=False)
assert torch.allclose(default, w_normal, atol=1e-2), "Default should use normal."
assert not torch.allclose(w_normal, wo_normal, atol=1e-2), (
"wo_normal should not use normal."
)
assert torch.allclose(w_normal, F.normalize(wo_normal, p=2, dim=-1), atol=1e-2), (
"w_normal should be close to normal(wo_normal)."
)
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
async def test_pooling_embed(server: RemoteOpenAIServer, model_name: str):
task = "embed"
response = requests.post(
server.url_for("pooling"),
json={
"model": model_name,
"input": input_text,
"encoding_format": "float",
"task": task,
},
)
poolings = PoolingResponse.model_validate(response.json())
assert len(poolings.data) == 1
assert len(poolings.data[0].data) == 384
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
@pytest.mark.parametrize(
"task", ["classify", "token_classify", "token_embed", "plugin"]
)
async def test_pooling_not_supported(
server: RemoteOpenAIServer, model_name: str, task: str
):
response = requests.post(
server.url_for("pooling"),
json={
"model": model_name,
"input": "test",
"encoding_format": "float",
"task": task,
},
)
assert response.json()["error"]["type"] == "BadRequestError"
if task == "plugin":
err_msg = "No IOProcessor plugin installed."
elif task == "token_embed":
err_msg = "Try switching the model's pooling_task via"
else:
err_msg = f"Unsupported task: {task!r}"
assert response.json()["error"]["message"].startswith(err_msg)
@@ -1,127 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Run `pytest tests/entrypoints/openai/test_embedding_dimensions.py`."""
import openai
import pytest
from tests.conftest import HfRunner
from tests.models.language.pooling.embed_utils import (
float_embeddings,
run_embedding_correctness_test,
)
from tests.models.utils import EmbedModelInfo
from tests.utils import ROCM_EXTRA_ARGS, RemoteOpenAIServer
from vllm.entrypoints.pooling.embed.protocol import EmbeddingResponse
MODELS = [
EmbedModelInfo("intfloat/multilingual-e5-small", is_matryoshka=False),
EmbedModelInfo(
"Snowflake/snowflake-arctic-embed-m-v1.5",
is_matryoshka=True,
matryoshka_dimensions=[256],
),
]
input_texts = [
"The chef prepared a delicious meal.",
]
@pytest.fixture(scope="module", params=MODELS)
def model_info(request):
return request.param
@pytest.fixture(scope="module", params=["bfloat16"])
def dtype(request):
return request.param
@pytest.fixture(scope="module")
def server(model_info, dtype: str):
args = [
"--runner",
"pooling",
# use half precision for speed and memory savings in CI environment
"--dtype",
dtype,
"--enforce-eager",
"--max-model-len",
"512",
] + ROCM_EXTRA_ARGS
if model_info.name == "Snowflake/snowflake-arctic-embed-m-v1.5":
# Manually enable Matryoshka Embeddings
args.extend(
["--trust_remote_code", "--hf_overrides", '{"matryoshka_dimensions":[256]}']
)
with RemoteOpenAIServer(model_info.name, args) as remote_server:
yield remote_server
@pytest.fixture(scope="module")
def hf_model(hf_runner, model_info, dtype: str):
with hf_runner(
model_info.name, dtype=dtype, is_sentence_transformer=True
) as hf_model:
yield hf_model
@pytest.mark.asyncio
async def test_matryoshka(
model_info: EmbedModelInfo, server: RemoteOpenAIServer, hf_model: HfRunner
):
client = server.get_async_client()
async def make_request_and_correctness_test(dimensions):
prompts = input_texts * 3
embedding_response = await client.embeddings.create(
model=model_info.name,
input=prompts,
dimensions=dimensions,
encoding_format="float",
)
embeddings = EmbeddingResponse.model_validate(
embedding_response.model_dump(mode="json")
)
assert embeddings.id is not None
assert len(embeddings.data) == 3
assert len(embeddings.data[0].embedding) > 0
assert embeddings.usage.completion_tokens == 0
assert embeddings.usage.prompt_tokens > 0
assert embeddings.usage.total_tokens > 0
if dimensions is not None:
assert len(embeddings.data[0].embedding) == dimensions
vllm_outputs = float_embeddings(embeddings)
run_embedding_correctness_test(hf_model, prompts, vllm_outputs, dimensions)
if model_info.is_matryoshka:
valid_dimensions: list[int | None] = [None]
if model_info.matryoshka_dimensions is not None:
valid_dimensions += model_info.matryoshka_dimensions[:2]
for dimensions in valid_dimensions:
await make_request_and_correctness_test(dimensions)
invalid_dimensions: list[int | None] = [-1]
if model_info.matryoshka_dimensions is not None:
assert 5 not in model_info.matryoshka_dimensions
invalid_dimensions.append(5)
for dimensions in invalid_dimensions:
with pytest.raises(openai.BadRequestError):
await make_request_and_correctness_test(dimensions)
else:
for dimensions in [None]:
await make_request_and_correctness_test(dimensions)
for dimensions in [-1, 16]:
with pytest.raises(openai.BadRequestError):
await make_request_and_correctness_test(dimensions)
@@ -1,452 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Test cases for long text embedding with automatic chunking mechanism.
This test suite validates vLLM's automatic chunking functionality for handling
text inputs that exceed the model's maximum token length, specifically targeting
the intfloat/multilingual-e5-small model (max token length: 512).
"""
import random
import openai
import pytest
import pytest_asyncio
from tests.utils import RemoteOpenAIServer
from vllm.entrypoints.pooling.embed.protocol import EmbeddingResponse
def _generate_random_text(word_count: int) -> str:
"""Generate deterministic text with approximately the specified word count."""
rng = random.Random(word_count)
# Common English words with focus on verbs and nouns for realistic text
common_words = [
# Essential articles and pronouns (minimal)
"the",
"and",
"you",
"they",
"this",
"that",
"these",
"those",
# Action verbs
"create",
"build",
"develop",
"design",
"implement",
"execute",
"analyze",
"process",
"generate",
"calculate",
"evaluate",
"optimize",
"transform",
"integrate",
"configure",
"deploy",
"monitor",
"manage",
"discover",
"explore",
"investigate",
"research",
"study",
"examine",
"improve",
"enhance",
"upgrade",
"modify",
"update",
"maintain",
"solve",
"resolve",
"handle",
"address",
"tackle",
"overcome",
"communicate",
"collaborate",
"coordinate",
"organize",
"plan",
"achieve",
"accomplish",
"complete",
"finish",
"deliver",
"provide",
# Technology and science nouns
"system",
"application",
"software",
"hardware",
"network",
"database",
"algorithm",
"model",
"framework",
"platform",
"interface",
"protocol",
"architecture",
"infrastructure",
"component",
"module",
"service",
"technology",
"innovation",
"solution",
"methodology",
"approach",
"artificial",
"intelligence",
"machine",
"learning",
"neural",
"network",
"computer",
"processor",
"memory",
"storage",
"computation",
"data",
"information",
"knowledge",
"insight",
"pattern",
"trend",
"analysis",
"research",
"development",
"engineering",
"science",
"mathematics",
"statistics",
"probability",
"optimization",
"performance",
"efficiency",
# General nouns
"project",
"team",
"organization",
"company",
"business",
"industry",
"market",
"customer",
"user",
"client",
"product",
"feature",
"function",
"requirement",
"specification",
"documentation",
"report",
"result",
"outcome",
"impact",
"benefit",
"advantage",
"challenge",
"problem",
"opportunity",
"strategy",
"goal",
"objective",
"target",
"milestone",
"process",
"procedure",
"workflow",
"pipeline",
"operation",
"task",
"activity",
"event",
"session",
"meeting",
"discussion",
"decision",
]
words = []
for _ in range(word_count):
words.append(rng.choice(common_words))
# Add some punctuation for more realistic text
text = " ".join(words)
# Add periods every 10-20 words
words_list = text.split()
result = []
for i, word in enumerate(words_list):
result.append(word)
if (i + 1) % rng.randint(10, 20) == 0 and i < len(words_list) - 1:
result[-1] += "."
return " ".join(result)
MODEL_NAME = "intfloat/multilingual-e5-small"
DTYPE = "bfloat16"
# Test text: Generate text with approximately 1500 words to exceed 1024 tokens
LONG_TEXT_1500_WORDS = _generate_random_text(1500)
# Test text: Generate text with approximately 2500 words to exceed 2048 tokens
LONG_TEXT_2500_WORDS = _generate_random_text(2500)
@pytest.fixture(scope="module")
def server_with_chunked_processing():
"""Start server with automatic chunking processing enabled."""
args = [
"--runner",
"pooling",
"--dtype",
DTYPE,
"--enforce-eager",
"--max-model-len",
"512", # Set smaller max_model_len to trigger chunking mechanism
"--pooler-config",
(
'{"pooling_type": "MEAN", "use_activation": true, '
'"enable_chunked_processing": true, "max_embed_len": 10000}'
),
"--gpu-memory-utilization",
"0.8",
]
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
yield remote_server
@pytest_asyncio.fixture
async def client_with_chunked_processing(server_with_chunked_processing):
"""Create async client with chunking processing support."""
async with server_with_chunked_processing.get_async_client() as async_client:
yield async_client
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
async def test_long_text_embedding_1500_chars(
client_with_chunked_processing: openai.AsyncOpenAI, model_name: str
):
"""Test embedding processing for ~1500 character long text
(~1028 tokens, exceeding 512 token limit)."""
# Verify text length
# Verify text has sufficient word count (approximately 1500 words)
word_count = len(LONG_TEXT_1500_WORDS.split())
assert word_count >= 1400, f"Test text word count insufficient: {word_count} words"
# Send embedding request
embedding_response = await client_with_chunked_processing.embeddings.create(
model=model_name,
input=[LONG_TEXT_1500_WORDS],
encoding_format="float",
)
# Verify response structure
embeddings = EmbeddingResponse.model_validate(
embedding_response.model_dump(mode="json")
)
assert embeddings.id is not None
assert len(embeddings.data) == 1
assert (
len(embeddings.data[0].embedding) == 384
) # multilingual-e5-small embedding dimension
assert embeddings.usage.completion_tokens == 0
# Due to chunked processing, token count should
# reflect actual processed tokens
# With ~1500 words, we expect roughly
# 1024+ tokens (exceeding 512 token limit)
# Should exceed single chunk limit of 512
assert embeddings.usage.prompt_tokens > 800
assert embeddings.usage.total_tokens == embeddings.usage.prompt_tokens
# Verify embedding vector validity
embedding_vector = embeddings.data[0].embedding
assert isinstance(embedding_vector, list)
assert all(isinstance(x, float) for x in embedding_vector), (
"Embedding vector should contain floats"
)
assert not all(x == 0 for x in embedding_vector), (
"Embedding vector should not be all zeros"
)
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
async def test_long_text_embedding_2500_chars(
client_with_chunked_processing: openai.AsyncOpenAI, model_name: str
):
"""Test embedding processing for ~2500 character long text
(~2048 tokens, requiring multiple chunks)."""
# Verify text length
# Verify text has sufficient word count (approximately 2500 words)
word_count = len(LONG_TEXT_2500_WORDS.split())
assert word_count >= 2300, f"Test text word count insufficient: {word_count} words"
# Send embedding request
embedding_response = await client_with_chunked_processing.embeddings.create(
model=model_name,
input=[LONG_TEXT_2500_WORDS],
encoding_format="float",
)
# Verify response structure
embeddings = EmbeddingResponse.model_validate(
embedding_response.model_dump(mode="json")
)
assert embeddings.id is not None
assert len(embeddings.data) == 1
assert (
len(embeddings.data[0].embedding) == 384
) # multilingual-e5-small embedding dimension
assert embeddings.usage.completion_tokens == 0
# Due to chunked processing, token count should
# reflect actual processed tokens
# With ~2500 words, we expect
# roughly 2048+ tokens (requiring multiple chunks)
# Should require multiple chunks for processing
assert embeddings.usage.prompt_tokens > 1500
assert embeddings.usage.total_tokens == embeddings.usage.prompt_tokens
# Verify embedding vector validity
embedding_vector = embeddings.data[0].embedding
assert isinstance(embedding_vector, list)
assert all(isinstance(x, float) for x in embedding_vector), (
"Embedding vector should contain floats"
)
assert not all(x == 0 for x in embedding_vector), (
"Embedding vector should not be all zeros"
)
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
async def test_batch_long_text_embedding(
client_with_chunked_processing: openai.AsyncOpenAI, model_name: str
):
"""Test batch long text embedding processing."""
input_texts = [
LONG_TEXT_1500_WORDS,
LONG_TEXT_2500_WORDS,
"This is a short text test.", # Short text for comparison
]
# Send batch embedding request
embedding_response = await client_with_chunked_processing.embeddings.create(
model=model_name,
input=input_texts,
encoding_format="float",
)
# Verify response structure
embeddings = EmbeddingResponse.model_validate(
embedding_response.model_dump(mode="json")
)
assert embeddings.id is not None
assert len(embeddings.data) == 3 # Three input texts
# Verify each embedding dimension
for i, embedding_data in enumerate(embeddings.data):
assert len(embedding_data.embedding) == 384
assert embedding_data.index == i
# Verify embedding vector validity
embedding_vector = embedding_data.embedding
assert all(isinstance(x, float) for x in embedding_vector)
assert not all(x == 0 for x in embedding_vector)
# Verify token usage
assert embeddings.usage.completion_tokens == 0
# Total token count should be very substantial
assert embeddings.usage.prompt_tokens > 1000
assert embeddings.usage.total_tokens == embeddings.usage.prompt_tokens
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
async def test_chunked_vs_normal_consistency(
client_with_chunked_processing: openai.AsyncOpenAI, model_name: str
):
"""Test consistency between chunked and
normal processing (using short text)."""
# Use a short text within the 512 token limit
short_text = (
"Artificial intelligence technology is changing our world, "
"bringing unprecedented opportunities and challenges."
)
# Send embedding request
embedding_response = await client_with_chunked_processing.embeddings.create(
model=model_name,
input=[short_text],
encoding_format="float",
)
# Verify response structure
embeddings = EmbeddingResponse.model_validate(
embedding_response.model_dump(mode="json")
)
assert embeddings.id is not None
assert len(embeddings.data) == 1
assert len(embeddings.data[0].embedding) == 384
assert embeddings.usage.completion_tokens == 0
# Short text should not require chunked processing
assert embeddings.usage.prompt_tokens < 512
assert embeddings.usage.total_tokens == embeddings.usage.prompt_tokens
# 验证embedding向量的有效性
embedding_vector = embeddings.data[0].embedding
assert isinstance(embedding_vector, list)
assert all(isinstance(x, float) for x in embedding_vector)
assert not all(x == 0 for x in embedding_vector)
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
async def test_chunked_processing_response_format(
client_with_chunked_processing: openai.AsyncOpenAI, model_name: str
):
"""Test response format and structure during chunked processing."""
# Test with long text to trigger chunking
embedding_response = await client_with_chunked_processing.embeddings.create(
model=model_name,
input=[LONG_TEXT_1500_WORDS],
encoding_format="float",
)
# Verify response structure
embeddings = EmbeddingResponse.model_validate(
embedding_response.model_dump(mode="json")
)
assert embeddings.id is not None
assert len(embeddings.data) == 1
assert embeddings.data[0].object == "embedding"
assert embeddings.data[0].index == 0
# Verify embedding vector properties
embedding_vector = embeddings.data[0].embedding
assert isinstance(embedding_vector, list)
import math
vector_norm = math.sqrt(sum(x * x for x in embedding_vector))
# Check that the vector is normalized
# (default behavior for most embedding models)
assert 0.8 < vector_norm < 1.2, (
f"Vector norm should be reasonable, actual: {vector_norm}"
)
@@ -1,211 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import pytest
import requests
from transformers import AutoProcessor
from tests.utils import VLLM_PATH, RemoteOpenAIServer
from vllm.entrypoints.pooling.embed.protocol import EmbeddingResponse
from vllm.multimodal.media import MediaWithBytes
from vllm.multimodal.utils import encode_image_url, fetch_image
MODEL_NAME = "TIGER-Lab/VLM2Vec-Full"
MAXIMUM_IMAGES = 2
vlm2vec_jinja_path = VLLM_PATH / "examples/pooling/embed/template/vlm2vec_phi3v.jinja"
assert vlm2vec_jinja_path.exists()
# Test different image extensions (JPG/PNG) and formats (gray/RGB/RGBA)
TEST_IMAGE_ASSETS = [
"2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
"Grayscale_8bits_palette_sample_image.png", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/Grayscale_8bits_palette_sample_image.png",
"1280px-Venn_diagram_rgb.svg.png", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/1280px-Venn_diagram_rgb.svg.png",
"RGBA_comp.png", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/RGBA_comp.png",
]
input_text = "The best thing about vLLM is that it supports many different models"
image_url = "https://vllm-public-assets.s3.us-west-2.amazonaws.com/multimodal_asset/cat_snow.jpg"
image_base64 = {"url": encode_image_url(fetch_image(image_url))}
@pytest.fixture(scope="module")
def server():
args = [
"--runner",
"pooling",
"--max-model-len",
"2048",
"--max-num-seqs",
"5",
"--enforce-eager",
"--trust-remote-code",
"--limit-mm-per-prompt",
json.dumps({"image": MAXIMUM_IMAGES}),
"--chat-template",
str(vlm2vec_jinja_path),
"--trust-request-mm-kwargs",
]
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
yield remote_server
@pytest.mark.parametrize("model_name", [MODEL_NAME])
def test_chat_text_request(server: RemoteOpenAIServer, model_name: str):
messages = [
{
"role": "user",
"content": input_text,
},
]
# note: vlm2vec_phi3v.jinja
# Embedding models should only embed one message at a time.
response = requests.post(
server.url_for("v1/embeddings"),
json={"model": model_name, "messages": messages},
)
response.raise_for_status()
output = EmbeddingResponse.model_validate(response.json())
assert len(output.data) == 1
assert output.model == MODEL_NAME
assert len(output.data[0].embedding) == 3072
assert output.usage.prompt_tokens == 14
@pytest.mark.parametrize("model_name", [MODEL_NAME])
def test_chat_image_url_request(server: RemoteOpenAIServer, model_name: str):
messages = [
{
"role": "user",
"content": [
{"type": "text", "text": "Represent the user's input."},
{"type": "image_url", "image_url": {"url": image_url}},
],
}
]
response = requests.post(
server.url_for("v1/embeddings"),
json={"model": model_name, "messages": messages},
)
response.raise_for_status()
output = EmbeddingResponse.model_validate(response.json())
assert len(output.data) == 1
assert output.model == MODEL_NAME
assert len(output.data[0].embedding) == 3072
assert output.usage.prompt_tokens == 767
@pytest.mark.parametrize("model_name", [MODEL_NAME])
def test_chat_image_base64_request(server: RemoteOpenAIServer, model_name: str):
messages = [
{
"role": "user",
"content": [
{"type": "text", "text": "Represent the user's input."},
{"type": "image_url", "image_url": image_base64},
],
}
]
response = requests.post(
server.url_for("v1/embeddings"),
json={"model": model_name, "messages": messages},
)
response.raise_for_status()
output = EmbeddingResponse.model_validate(response.json())
assert len(output.data) == 1
assert output.model == MODEL_NAME
assert len(output.data[0].embedding) == 3072
assert output.usage.prompt_tokens == 767
@pytest.mark.parametrize("model_name", [MODEL_NAME])
def test_chat_image_with_media_io_kwargs(server: RemoteOpenAIServer, model_name: str):
rgba_image_url = (
"https://vllm-public-assets.s3.us-west-2.amazonaws.com"
"/vision_model_images/RGBA_comp.png"
)
messages = [
{
"role": "user",
"content": [
{"type": "text", "text": "Represent the user's input."},
{"type": "image_url", "image_url": {"url": rgba_image_url}},
],
}
]
response = requests.post(
server.url_for("v1/embeddings"),
json={
"model": model_name,
"messages": messages,
"media_io_kwargs": {
"image": {"rgba_background_color": [0, 0, 0]},
},
},
)
response.raise_for_status()
output = EmbeddingResponse.model_validate(response.json())
assert len(output.data) == 1
assert len(output.data[0].embedding) == 3072
def get_hf_prompt_tokens(model_name, content, image_url):
processor = AutoProcessor.from_pretrained(
model_name, trust_remote_code=True, num_crops=4
)
placeholder = "<|image_1|> "
prompt = f"{placeholder}{content}"
image = fetch_image(image_url)
# Unwrap MediaWithBytes if present
if isinstance(image, MediaWithBytes):
image = image.media
images = [image]
inputs = processor(prompt, images, return_tensors="pt")
return inputs.input_ids.shape[1]
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
@pytest.mark.parametrize("image_url", TEST_IMAGE_ASSETS, indirect=True)
async def test_image_embedding(
server: RemoteOpenAIServer, model_name: str, image_url: str
):
content_text = "Represent the given image."
messages = [
{
"role": "user",
"content": [
{"type": "image_url", "image_url": {"url": image_url}},
{"type": "text", "text": content_text},
],
}
]
response = requests.post(
server.url_for("v1/embeddings"),
json={"model": model_name, "messages": messages, "encoding_format": "float"},
)
response.raise_for_status()
embeddings = EmbeddingResponse.model_validate(response.json())
hf_prompt_tokens = get_hf_prompt_tokens(model_name, content_text, image_url)
assert embeddings.id is not None
assert len(embeddings.data) == 1
assert len(embeddings.data[0].embedding) == 3072
assert embeddings.usage.completion_tokens == 0
assert embeddings.usage.prompt_tokens == hf_prompt_tokens
assert embeddings.usage.total_tokens == hf_prompt_tokens
@@ -1,223 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for Cohere embed protocol: build_typed_embeddings and its
underlying packing helpers, plus Cohere-specific serving helpers."""
import struct
from unittest.mock import Mock
import numpy as np
import pybase64 as base64
import pytest
from vllm.config import ModelConfig, PoolerConfig
from vllm.entrypoints.pooling.embed.protocol import (
EmbeddingCompletionRequest,
build_typed_embeddings,
)
@pytest.fixture
def sample_embeddings() -> list[list[float]]:
return [
[0.1, -0.2, 0.3, -0.4, 0.5, -0.6, 0.7, -0.8],
[-0.05, 0.15, -0.25, 0.35, -0.45, 0.55, -0.65, 0.75],
]
class TestBuildTypedEmbeddingsFloat:
def test_float_passthrough(self, sample_embeddings: list[list[float]]):
result = build_typed_embeddings(sample_embeddings, ["float"])
assert result.float == sample_embeddings
assert result.binary is None
def test_empty_input(self):
result = build_typed_embeddings([], ["float"])
assert result.float == []
class TestBuildTypedEmbeddingsBinary:
def test_binary_packing(self):
# 8 values: positive->1, negative->0 => bits: 10101010 = 0xAA = 170
# signed: 170 - 128 = 42
embs = [[1.0, -1.0, 1.0, -1.0, 1.0, -1.0, 1.0, -1.0]]
result = build_typed_embeddings(embs, ["binary"])
assert result.binary is not None
assert result.binary[0] == [42]
def test_ubinary_packing(self):
embs = [[1.0, -1.0, 1.0, -1.0, 1.0, -1.0, 1.0, -1.0]]
result = build_typed_embeddings(embs, ["ubinary"])
assert result.ubinary is not None
assert result.ubinary[0] == [170] # 0b10101010
def test_binary_all_positive(self):
embs = [[0.1] * 8]
result = build_typed_embeddings(embs, ["binary"])
assert result.binary is not None
# all bits = 1 => 0xFF = 255, signed: 255 - 128 = 127
assert result.binary[0] == [127]
def test_binary_all_negative(self):
embs = [[-0.1] * 8]
result = build_typed_embeddings(embs, ["binary"])
assert result.binary is not None
# all bits = 0, signed: 0 - 128 = -128
assert result.binary[0] == [-128]
def test_binary_dimension_is_eighth(self, sample_embeddings: list[list[float]]):
result = build_typed_embeddings(sample_embeddings, ["binary"])
assert result.binary is not None
for orig, packed in zip(sample_embeddings, result.binary):
assert len(packed) == len(orig) // 8
def test_zero_treated_as_positive(self):
embs = [[0.0] * 8]
result = build_typed_embeddings(embs, ["binary"])
assert result.binary is not None
# 0.0 >= 0 is True, so bit=1 for all => 127 (signed)
assert result.binary[0] == [127]
def test_negative_zero_treated_as_positive(self):
"""Packing follows ``value >= 0``, not the IEEE-754 sign bit.
``-0.0 >= 0`` is True, so these bits must be set. An implementation
reading the sign bit instead would pack -128 here.
"""
result = build_typed_embeddings([[-0.0] * 8], ["binary"])
assert result.binary is not None
assert result.binary[0] == [127]
def test_negative_denormal_treated_as_negative(self):
"""Sign is decided at float64 precision.
The smallest float64 denormal underflows to -0.0 in float32, where
``>= 0`` is True; narrowing the comparison would wrongly pack 127.
"""
result = build_typed_embeddings([[-5e-324] * 8], ["binary"])
assert result.binary is not None
assert result.binary[0] == [-128]
def test_empty_input(self):
result = build_typed_embeddings([], ["binary", "ubinary"])
assert result.binary == []
assert result.ubinary == []
@pytest.mark.parametrize("embs", [[0.1] * 8, [[[0.1] * 8] * 2]], ids=["1d", "3d"])
def test_non_2d_input_raises(self, embs):
"""A malformed batch must raise rather than pack along the last axis.
Packing a 1D or 3D input would silently emit wrongly shaped
``binary``/``ubinary`` values instead of reporting an error.
"""
with pytest.raises(ValueError, match="2D batch"):
build_typed_embeddings(embs, ["binary"])
def test_non_multiple_of_8_raises(self):
embs = [[0.1] * 7]
with pytest.raises(ValueError, match="multiple of 8"):
build_typed_embeddings(embs, ["binary"])
def test_ubinary_non_multiple_of_8_raises(self):
embs = [[0.1] * 10]
with pytest.raises(ValueError, match="multiple of 8"):
build_typed_embeddings(embs, ["ubinary"])
class TestBuildTypedEmbeddingsBase64:
def test_base64_roundtrip(self, sample_embeddings: list[list[float]]):
result = build_typed_embeddings(sample_embeddings, ["base64"])
assert result.base64 is not None
assert len(result.base64) == 2
for orig, b64_str in zip(sample_embeddings, result.base64):
decoded = base64.b64decode(b64_str)
n = len(orig)
values = struct.unpack(f"<{n}f", decoded)
np.testing.assert_allclose(orig, values, rtol=1e-5)
def test_base64_byte_length(self):
embs = [[0.1, 0.2, 0.3]]
result = build_typed_embeddings(embs, ["base64"])
assert result.base64 is not None
raw = base64.b64decode(result.base64[0])
assert len(raw) == 3 * 4 # 3 floats * 4 bytes each
class TestBuildTypedEmbeddingsMultiple:
def test_all_types_at_once(self, sample_embeddings: list[list[float]]):
result = build_typed_embeddings(
sample_embeddings,
["float", "binary", "ubinary", "base64"],
)
assert result.float is not None
assert result.binary is not None
assert result.ubinary is not None
assert result.base64 is not None
def test_subset_types(self, sample_embeddings: list[list[float]]):
result = build_typed_embeddings(sample_embeddings, ["float", "binary"])
assert result.float is not None
assert result.binary is not None
assert result.ubinary is None
assert result.base64 is None
def test_unknown_type_ignored(self, sample_embeddings: list[list[float]]):
result = build_typed_embeddings(sample_embeddings, ["float", "unknown_type"])
assert result.float is not None
def _embedding_model_config(
*,
max_model_len: int = 128,
max_embed_len: int | None = None,
enable_chunked_processing: bool = False,
) -> Mock:
model_config = Mock(spec=ModelConfig)
model_config.max_model_len = max_model_len
model_config.encoder_config = None
model_config.pooler_config = PoolerConfig(
seq_pooling_type="CLS",
max_embed_len=max_embed_len,
enable_chunked_processing=enable_chunked_processing,
)
return model_config
class TestMaxEmbedLenCapsInput:
"""``build_tok_params`` used to enforce ``max_embed_len`` indirectly, by
setting ``max_output_tokens = max_model_len - max_embed_len`` and relying on
``max_input_tokens == max_total_tokens - max_output_tokens``. The cap is now
expressed directly as ``max_total_tokens``."""
def test_non_chunked_max_embed_len_caps_input(self):
cfg = _embedding_model_config(max_model_len=128, max_embed_len=64)
tok_params = EmbeddingCompletionRequest(model="m", input="hi").build_tok_params(
cfg
)
assert tok_params.max_input_tokens == 64
assert tok_params.max_output_tokens == 0
def test_chunked_max_embed_len_caps_input(self):
cfg = _embedding_model_config(
max_model_len=128, max_embed_len=64, enable_chunked_processing=True
)
tok_params = EmbeddingCompletionRequest(model="m", input="hi").build_tok_params(
cfg
)
assert tok_params.max_input_tokens == 64
assert tok_params.max_output_tokens == 0
def test_no_max_embed_len_falls_back_to_model_len(self):
cfg = _embedding_model_config(max_model_len=128)
tok_params = EmbeddingCompletionRequest(model="m", input="hi").build_tok_params(
cfg
)
assert tok_params.max_input_tokens == 128
assert tok_params.max_output_tokens == 0
@@ -1,64 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import weakref
import pytest
import torch
from tests.models.utils import softmax
from vllm import LLM, PoolingParams
MODEL_NAME = "internlm/internlm2-1_8b-reward"
prompts = ["The chef prepared a delicious meal."]
@pytest.fixture(scope="module")
def llm(vllm_runner):
with vllm_runner(
MODEL_NAME,
max_model_len=None,
max_num_batched_tokens=32768,
tensor_parallel_size=1,
gpu_memory_utilization=0.75,
enforce_eager=True,
trust_remote_code=True,
seed=0,
enable_chunked_prefill=None,
) as runner:
# pytest caches yielded fixtures until after teardown, so use a proxy to
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
yield weakref.proxy(runner.llm)
@pytest.mark.skip_global_cleanup
def test_config(llm: LLM):
vllm_config = llm.llm_engine.vllm_config
assert vllm_config.cache_config.enable_prefix_caching
assert vllm_config.scheduler_config.enable_chunked_prefill
def test_pooling_params(llm: LLM):
def get_outputs(use_activation):
outputs = llm.encode(
prompts,
pooling_params=PoolingParams(use_activation=use_activation),
pooling_task="token_classify",
use_tqdm=False,
)
return torch.cat([x.outputs.data for x in outputs])
default = get_outputs(use_activation=None)
w_activation = get_outputs(use_activation=True)
wo_activation = get_outputs(use_activation=False)
assert torch.allclose(default, w_activation, atol=1e-2), (
"Default should use activation."
)
assert not torch.allclose(w_activation, wo_activation, atol=1e-2), (
"wo_activation should not use activation."
)
assert torch.allclose(softmax(wo_activation), w_activation, atol=1e-2), (
"w_activation should be close to activation(wo_activation)."
)
@@ -1,570 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import numpy as np
import pybase64 as base64
import pytest
import requests
import torch
from tests.models.utils import check_embeddings_close
from tests.utils import RemoteOpenAIServer
from vllm.entrypoints.pooling.pooling.protocol import PoolingResponse
from vllm.entrypoints.pooling.utils import (
MetadataItem,
build_metadata_items,
decode_pooling_output,
)
from vllm.renderers.chat_utils import ChatCompletionMessageParam
from vllm.tokenizers import get_tokenizer
from vllm.utils.serial_utils import EMBED_DTYPES, ENDIANNESS, binary2tensor
MODEL_NAME = "internlm/internlm2-1_8b-reward"
DUMMY_CHAT_TEMPLATE = """{% for message in messages %}{{message['role'] + ': ' + message['content'] + '\\n'}}{% endfor %}""" # noqa: E501
input_text = "The chef prepared a delicious meal."
input_tokens = [1, 918, 29981, 10166, 395, 18067, 15265, 281]
@pytest.fixture(scope="module")
def server():
args = [
"--runner",
"pooling",
# use half precision for speed and memory savings in CI environment
"--dtype",
"bfloat16",
"--enforce-eager",
"--max-model-len",
"512",
"--chat-template",
DUMMY_CHAT_TEMPLATE,
"--trust-remote-code",
]
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
yield remote_server
@pytest.mark.parametrize("model_name", [MODEL_NAME])
def test_basic(server: RemoteOpenAIServer, model_name: str):
# test /v1/models
response = requests.get(server.url_for("/v1/models"))
served_model = response.json()["data"][0]["id"]
assert served_model == MODEL_NAME
# test /tokenize
response = requests.post(
server.url_for("/tokenize"),
json={"model": model_name, "prompt": input_text},
)
assert response.json()["tokens"] == input_tokens
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
def test_completion_request(server: RemoteOpenAIServer, model_name: str):
# test input: str
response = requests.post(
server.url_for("pooling"),
json={"model": model_name, "input": input_text, "encoding_format": "float"},
)
response.raise_for_status()
poolings = PoolingResponse.model_validate(response.json())
assert poolings.id is not None
assert len(poolings.data) == 1
assert len(poolings.data[0].data) == len(input_tokens)
assert poolings.usage.completion_tokens == 0
assert poolings.usage.prompt_tokens == len(input_tokens)
assert poolings.usage.total_tokens == len(input_tokens)
# test input: list[int]
response = requests.post(
server.url_for("pooling"),
json={"model": model_name, "input": input_tokens, "encoding_format": "float"},
)
response.raise_for_status()
poolings = PoolingResponse.model_validate(response.json())
assert poolings.id is not None
assert len(poolings.data) == 1
assert len(poolings.data[0].data) == len(input_tokens)
assert poolings.usage.completion_tokens == 0
assert poolings.usage.prompt_tokens == len(input_tokens)
assert poolings.usage.total_tokens == len(input_tokens)
@pytest.mark.parametrize("model_name", [MODEL_NAME])
def test_completion_request_batched(server: RemoteOpenAIServer, model_name: str):
N = 10
input_texts = [input_text] * N
response = requests.post(
server.url_for("pooling"),
json={"model": model_name, "input": input_texts, "encoding_format": "float"},
)
response.raise_for_status()
poolings = PoolingResponse.model_validate(response.json())
assert poolings.id is not None
assert len(poolings.data) == N
assert len(poolings.data[0].data) == len(input_tokens)
assert poolings.usage.completion_tokens == 0
assert poolings.usage.prompt_tokens == len(input_tokens) * N
assert poolings.usage.total_tokens == len(input_tokens) * N
# test list[list[int]]
response = requests.post(
server.url_for("pooling"),
json={
"model": model_name,
"input": [input_tokens] * N,
"encoding_format": "float",
},
)
response.raise_for_status()
poolings = PoolingResponse.model_validate(response.json())
assert poolings.id is not None
assert len(poolings.data) == N
assert len(poolings.data[0].data) == len(input_tokens)
assert poolings.usage.completion_tokens == 0
assert poolings.usage.prompt_tokens == len(input_tokens) * N
assert poolings.usage.total_tokens == len(input_tokens) * N
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
async def test_chat_request(server: RemoteOpenAIServer, model_name: str):
messages: list[ChatCompletionMessageParam] = [
{
"role": "user",
"content": "The cat sat on the mat.",
},
{
"role": "assistant",
"content": "A feline was resting on a rug.",
},
{
"role": "user",
"content": "Stars twinkle brightly in the night sky.",
},
]
# test chat request basic usage
chat_response = requests.post(
server.url_for("pooling"),
json={
"model": model_name,
"messages": messages,
"encoding_format": "float",
},
)
chat_response.raise_for_status()
chat_poolings = PoolingResponse.model_validate(chat_response.json())
tokenizer = get_tokenizer(tokenizer_name=model_name, trust_remote_code=True)
prompt = tokenizer.apply_chat_template(
messages,
chat_template=DUMMY_CHAT_TEMPLATE,
add_generation_prompt=True,
continue_final_message=False,
tokenize=False,
)
completions_response = requests.post(
server.url_for("pooling"),
json={
"model": model_name,
"input": prompt,
"encoding_format": "float",
# To be consistent with chat
"add_special_tokens": False,
},
)
completions_response.raise_for_status()
completion_poolings = PoolingResponse.model_validate(completions_response.json())
assert chat_poolings.id is not None
assert completion_poolings.id is not None
assert chat_poolings.created <= completion_poolings.created
assert chat_poolings.model_dump(exclude={"id", "created"}) == (
completion_poolings.model_dump(exclude={"id", "created"})
)
# test add_generation_prompt
response = requests.post(
server.url_for("pooling"),
json={"model": model_name, "messages": messages, "add_generation_prompt": True},
)
response.raise_for_status()
output = PoolingResponse.model_validate(response.json())
assert output.object == "list"
assert len(output.data) == 1
assert output.model == MODEL_NAME
assert output.usage.prompt_tokens == 33
# test continue_final_message
# The continue_final_message parameter doesn't seem to be working with this model.
response = requests.post(
server.url_for("pooling"),
json={
"model": model_name,
"messages": messages,
"continue_final_message": True,
},
)
response.raise_for_status()
output = PoolingResponse.model_validate(response.json())
assert output.object == "list"
assert len(output.data) == 1
assert output.model == MODEL_NAME
assert output.usage.prompt_tokens == 33
# test add_special_tokens
response = requests.post(
server.url_for("pooling"),
json={"model": model_name, "messages": messages, "add_special_tokens": True},
)
response.raise_for_status()
output = PoolingResponse.model_validate(response.json())
assert output.object == "list"
assert len(output.data) == 1
assert output.model == MODEL_NAME
assert output.usage.prompt_tokens == 34
# test continue_final_message with add_generation_prompt
response = requests.post(
server.url_for("pooling"),
json={
"model": model_name,
"messages": messages,
"continue_final_message": True,
"add_generation_prompt": True,
},
)
assert (
"Cannot set both `continue_final_message` and `add_generation_prompt` to True."
in response.json()["error"]["message"]
)
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
async def test_batch_base64_pooling(server: RemoteOpenAIServer, model_name: str):
input_texts = [
"Hello my name is",
"The best thing about vLLM is that it supports many different models",
]
float_response = requests.post(
server.url_for("pooling"),
json={
"input": input_texts,
"model": model_name,
"encoding_format": "float",
},
)
float_response.raise_for_status()
responses_float = PoolingResponse.model_validate(float_response.json())
float_data = [np.array(d.data).squeeze(-1).tolist() for d in responses_float.data]
base64_response = requests.post(
server.url_for("pooling"),
json={
"input": input_texts,
"model": model_name,
"encoding_format": "base64",
},
)
base64_response.raise_for_status()
responses_base64 = PoolingResponse.model_validate(base64_response.json())
decoded_responses_base64_data = []
for data in responses_base64.data:
decoded_responses_base64_data.append(
np.frombuffer(base64.b64decode(data.data), dtype="float32").tolist()
)
check_embeddings_close(
embeddings_0_lst=float_data,
embeddings_1_lst=decoded_responses_base64_data,
name_0="float32",
name_1="base64",
)
# Default response is float32 decoded from base64 by OpenAI Client
default_response = requests.post(
server.url_for("pooling"),
json={
"input": input_texts,
"model": model_name,
},
)
default_response.raise_for_status()
responses_default = PoolingResponse.model_validate(default_response.json())
default_data = [
np.array(d.data).squeeze(-1).tolist() for d in responses_default.data
]
check_embeddings_close(
embeddings_0_lst=float_data,
embeddings_1_lst=default_data,
name_0="float32",
name_1="default",
)
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
async def test_base64_embed_dtype_and_endianness(
server: RemoteOpenAIServer, model_name: str
):
input_texts = [input_text] * 3
url = server.url_for("pooling")
float_response = requests.post(
url,
json={
"model": model_name,
"input": input_texts,
"encoding_format": "float",
},
)
responses_float = PoolingResponse.model_validate(float_response.json())
float_data = [np.array(d.data).squeeze(-1).tolist() for d in responses_float.data]
for embed_dtype in EMBED_DTYPES:
for endianness in ENDIANNESS:
responses_base64 = requests.post(
url,
json={
"model": model_name,
"input": input_texts,
"encoding_format": "base64",
"embed_dtype": embed_dtype,
"endianness": endianness,
},
)
base64_data = []
for data in responses_base64.json()["data"]:
binary = base64.b64decode(data["data"])
tensor = binary2tensor(binary, (-1,), embed_dtype, endianness)
base64_data.append(tensor.to(torch.float32).tolist())
check_embeddings_close(
embeddings_0_lst=float_data,
embeddings_1_lst=base64_data,
name_0="float_data",
name_1="base64_data",
tol=1e-2,
)
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
async def test_bytes_embed_dtype_and_endianness(
server: RemoteOpenAIServer, model_name: str
):
input_texts = [input_text] * 3
url = server.url_for("pooling")
float_response = requests.post(
url,
json={
"model": model_name,
"input": input_texts,
"encoding_format": "float",
},
)
responses_float = PoolingResponse.model_validate(float_response.json())
float_data = [np.array(d.data).squeeze(-1).tolist() for d in responses_float.data]
for embed_dtype in EMBED_DTYPES:
for endianness in ENDIANNESS:
responses_bytes = requests.post(
url,
json={
"model": model_name,
"input": input_texts,
"encoding_format": "bytes",
"embed_dtype": embed_dtype,
"endianness": endianness,
},
)
metadata = json.loads(responses_bytes.headers["metadata"])
body = responses_bytes.content
items = [MetadataItem(**x) for x in metadata["data"]]
bytes_data = decode_pooling_output(items=items, body=body)
bytes_data = [x.to(torch.float32).view(-1).tolist() for x in bytes_data]
check_embeddings_close(
embeddings_0_lst=float_data,
embeddings_1_lst=bytes_data,
name_0="float_data",
name_1="bytes_data",
tol=1e-2,
)
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
async def test_bytes_only_embed_dtype_and_endianness(
server: RemoteOpenAIServer, model_name: str
):
input_texts = [input_text] * 3
url = server.url_for("pooling")
float_response = requests.post(
url,
json={
"model": model_name,
"input": input_texts,
"encoding_format": "float",
},
)
responses_float = PoolingResponse.model_validate(float_response.json())
float_data = [np.array(d.data).squeeze(-1).tolist() for d in responses_float.data]
n_tokens = responses_float.usage.prompt_tokens // len(input_texts)
for embed_dtype in EMBED_DTYPES:
for endianness in ENDIANNESS:
responses_bytes = requests.post(
url,
json={
"model": model_name,
"input": input_texts,
"encoding_format": "bytes_only",
"embed_dtype": embed_dtype,
"endianness": endianness,
},
)
assert "metadata" not in responses_bytes.headers
body = responses_bytes.content
items = build_metadata_items(
embed_dtype=embed_dtype,
endianness=endianness,
shape=(n_tokens, 1),
n_request=len(input_texts),
)
bytes_data = decode_pooling_output(items=items, body=body)
bytes_data = [x.to(torch.float32).view(-1).tolist() for x in bytes_data]
check_embeddings_close(
embeddings_0_lst=float_data,
embeddings_1_lst=bytes_data,
name_0="float_data",
name_1="bytes_data",
tol=1e-2,
)
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
@pytest.mark.parametrize("param_name", ["encoding_format", "embed_dtype", "endianness"])
async def test_params_not_supported(
server: RemoteOpenAIServer, model_name: str, param_name: str
):
responses_base64 = requests.post(
server.url_for("pooling"),
json={
"model": model_name,
"input": input_text,
"encoding_format": "base64",
param_name: f"bad_{param_name}",
},
)
assert responses_base64.status_code == 400
assert "literal_error" in responses_base64.json()["error"]["message"]
assert f"bad_{param_name}" in responses_base64.json()["error"]["message"]
@pytest.mark.asyncio
async def test_invocations_chat_request(server: RemoteOpenAIServer):
request_args = {
"model": MODEL_NAME,
"input": input_text,
"encoding_format": "float",
}
completion_response = requests.post(server.url_for("pooling"), json=request_args)
completion_response.raise_for_status()
invocation_response = requests.post(
server.url_for("invocations"), json=request_args
)
invocation_response.raise_for_status()
completion_output = completion_response.json()
invocation_output = invocation_response.json()
assert completion_output.keys() == invocation_output.keys()
for completion_data, invocation_data in zip(
completion_output["data"], invocation_output["data"]
):
assert completion_data.keys() == invocation_data.keys()
check_embeddings_close(
embeddings_0_lst=completion_data["data"],
embeddings_1_lst=invocation_data["data"],
name_0="completion",
name_1="invocation",
)
@pytest.mark.asyncio
async def test_invocations_conversation_chat_request(server: RemoteOpenAIServer):
messages: list[ChatCompletionMessageParam] = [
{
"role": "user",
"content": "The cat sat on the mat.",
},
{
"role": "assistant",
"content": "A feline was resting on a rug.",
},
{
"role": "user",
"content": "Stars twinkle brightly in the night sky.",
},
]
request_args = {
"model": MODEL_NAME,
"messages": messages,
"encoding_format": "float",
}
chat_response = requests.post(server.url_for("pooling"), json=request_args)
chat_response.raise_for_status()
invocation_response = requests.post(
server.url_for("invocations"), json=request_args
)
invocation_response.raise_for_status()
chat_output = chat_response.json()
invocation_output = invocation_response.json()
assert chat_output.keys() == invocation_output.keys()
for chat_data, invocation_data in zip(
chat_output["data"], invocation_output["data"]
):
assert chat_data.keys() == invocation_data.keys()
check_embeddings_close(
embeddings_0_lst=chat_data["data"],
embeddings_1_lst=invocation_data["data"],
name_0="chat",
name_1="invocation",
)
@@ -1,101 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import weakref
import pytest
from tests.entrypoints.pooling.scoring.util import EncoderScoringHfRunner
MODEL_NAME = "intfloat/multilingual-e5-small"
PROMPT = "The chef prepared a delicious meal."
EMBEDDING_SIZE = 384
TEXTS_1 = [
"What is the capital of France?",
"What is the capital of Germany?",
]
TEXTS_2 = [
"The capital of France is Paris.",
"The capital of Germany is Berlin.",
]
DTYPE = "half"
@pytest.fixture(scope="module")
def llm(vllm_runner):
with vllm_runner(
MODEL_NAME,
max_model_len=None,
max_num_batched_tokens=32768,
tensor_parallel_size=1,
gpu_memory_utilization=0.75,
enforce_eager=True,
seed=0,
enable_chunked_prefill=None,
) as runner:
# pytest caches yielded fixtures until after teardown, so use a proxy to
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
yield weakref.proxy(runner.llm)
@pytest.fixture(scope="module")
def hf_model():
return EncoderScoringHfRunner(MODEL_NAME)
@pytest.mark.skip_global_cleanup
def test_1_to_1(llm, hf_model):
text_pair = [TEXTS_1[0], TEXTS_2[0]]
hf_outputs = hf_model.predict([text_pair]).tolist()
vllm_outputs = [
output.outputs.score for output in llm.score(text_pair[0], text_pair[1])
]
assert len(vllm_outputs) == 1
assert len(hf_outputs) == 1
assert hf_outputs[0] == pytest.approx(vllm_outputs[0], rel=0.01)
@pytest.mark.skip_global_cleanup
def test_1_to_n(llm, hf_model):
text_pairs = [
[TEXTS_1[0], TEXTS_2[0]],
[TEXTS_1[0], TEXTS_2[1]],
]
hf_outputs = hf_model.predict(text_pairs).tolist()
vllm_outputs = [output.outputs.score for output in llm.score(TEXTS_1[0], TEXTS_2)]
assert len(vllm_outputs) == 2
assert len(hf_outputs) == 2
assert hf_outputs[0] == pytest.approx(vllm_outputs[0], rel=0.01)
assert hf_outputs[1] == pytest.approx(vllm_outputs[1], rel=0.01)
@pytest.mark.skip_global_cleanup
def test_n_to_n(llm, hf_model):
text_pairs = [
[TEXTS_1[0], TEXTS_2[0]],
[TEXTS_1[1], TEXTS_2[1]],
]
hf_outputs = hf_model.predict(text_pairs).tolist()
vllm_outputs = [output.outputs.score for output in llm.score(TEXTS_1, TEXTS_2)]
assert len(vllm_outputs) == 2
assert len(hf_outputs) == 2
assert hf_outputs[0] == pytest.approx(vllm_outputs[0], rel=0.01)
assert hf_outputs[1] == pytest.approx(vllm_outputs[1], rel=0.01)
def test_embed(llm):
outputs = llm.encode(PROMPT, pooling_task="embed", use_tqdm=False)
assert len(outputs) == 1
assert len(outputs[0].outputs.data) == EMBEDDING_SIZE
@@ -1,384 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import requests
from tests.entrypoints.pooling.scoring.util import EncoderScoringHfRunner
from tests.utils import RemoteOpenAIServer
from vllm.entrypoints.pooling.pooling.protocol import PoolingResponse
from vllm.entrypoints.pooling.scoring.protocol import RerankResponse, ScoreResponse
MODEL_NAME = "BAAI/bge-base-en-v1.5"
input_text = "This product was excellent and exceeded my expectations"
DTYPE = "half"
EMBEDDING_SIZE = 768
TEXTS_1 = [
"What is the capital of France?",
"What is the capital of Germany?",
]
TEXTS_2 = [
"The capital of France is Paris.",
"The capital of Germany is Berlin.",
]
@pytest.fixture(scope="module")
def server():
args = ["--enforce-eager", "--max-model-len", "100", "--dtype", DTYPE]
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
yield remote_server
@pytest.fixture(scope="module")
def hf_model():
return EncoderScoringHfRunner(MODEL_NAME)
@pytest.mark.asyncio
async def test_score_api_queries_str_1_documents_str_1(
hf_model, server: RemoteOpenAIServer
):
score_response = requests.post(
server.url_for("score"),
json={
"model": MODEL_NAME,
"queries": TEXTS_1[0],
"documents": TEXTS_2[0],
},
)
score_response.raise_for_status()
score = ScoreResponse.model_validate(score_response.json())
assert score.id is not None
assert score.data is not None
assert len(score.data) == 1
vllm_outputs = [d.score for d in score.data]
hf_outputs = hf_model.predict([[TEXTS_1[0], TEXTS_2[0]]]).tolist()
for i in range(len(vllm_outputs)):
assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01)
@pytest.mark.asyncio
async def test_score_api_queries_str_1_documents_str_n(
hf_model, server: RemoteOpenAIServer
):
text_pairs = [
[TEXTS_1[0], TEXTS_2[0]],
[TEXTS_1[0], TEXTS_2[1]],
]
score_response = requests.post(
server.url_for("score"),
json={
"model": MODEL_NAME,
"queries": TEXTS_1[0],
"documents": TEXTS_2,
},
)
score_response.raise_for_status()
score = ScoreResponse.model_validate(score_response.json())
assert score.id is not None
assert score.data is not None
assert len(score.data) == 2
vllm_outputs = [d.score for d in score.data]
hf_outputs = hf_model.predict(text_pairs).tolist()
for i in range(len(vllm_outputs)):
assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01)
@pytest.mark.asyncio
async def test_score_api_queries_str_n_documents_str_n(
hf_model, server: RemoteOpenAIServer
):
text_pairs = [
[TEXTS_1[0], TEXTS_2[0]],
[TEXTS_1[1], TEXTS_2[1]],
]
score_response = requests.post(
server.url_for("score"),
json={
"model": MODEL_NAME,
"queries": TEXTS_1,
"documents": TEXTS_2,
},
)
score_response.raise_for_status()
score = ScoreResponse.model_validate(score_response.json())
assert score.id is not None
assert score.data is not None
assert len(score.data) == 2
vllm_outputs = [d.score for d in score.data]
hf_outputs = hf_model.predict(text_pairs).tolist()
for i in range(len(vllm_outputs)):
assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01)
@pytest.mark.asyncio
async def test_score_api_queries_vs_items(hf_model, server: RemoteOpenAIServer):
text_pairs = [
[TEXTS_1[0], TEXTS_2[0]],
[TEXTS_1[1], TEXTS_2[1]],
]
score_response = requests.post(
server.url_for("score"),
json={
"model": MODEL_NAME,
"queries": TEXTS_1,
"items": TEXTS_2,
},
)
score_response.raise_for_status()
score = ScoreResponse.model_validate(score_response.json())
assert score.id is not None
assert score.data is not None
assert len(score.data) == 2
vllm_outputs = [d.score for d in score.data]
hf_outputs = hf_model.predict(text_pairs).tolist()
for i in range(len(vllm_outputs)):
assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01)
@pytest.mark.asyncio
async def test_score_api_text_1_vs_text_2(hf_model, server: RemoteOpenAIServer):
text_pairs = [
[TEXTS_1[0], TEXTS_2[0]],
[TEXTS_1[1], TEXTS_2[1]],
]
score_response = requests.post(
server.url_for("score"),
json={
"model": MODEL_NAME,
"text_1": TEXTS_1,
"text_2": TEXTS_2,
},
)
score_response.raise_for_status()
score = ScoreResponse.model_validate(score_response.json())
assert score.id is not None
assert score.data is not None
assert len(score.data) == 2
vllm_outputs = [d.score for d in score.data]
hf_outputs = hf_model.predict(text_pairs).tolist()
for i in range(len(vllm_outputs)):
assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01)
@pytest.mark.asyncio
async def test_score_api_data_1_vs_data_2(hf_model, server: RemoteOpenAIServer):
text_pairs = [
[TEXTS_1[0], TEXTS_2[0]],
[TEXTS_1[1], TEXTS_2[1]],
]
score_response = requests.post(
server.url_for("score"),
json={
"model": MODEL_NAME,
"data_1": TEXTS_1,
"data_2": TEXTS_2,
},
)
score_response.raise_for_status()
score = ScoreResponse.model_validate(score_response.json())
assert score.id is not None
assert score.data is not None
assert len(score.data) == 2
vllm_outputs = [d.score for d in score.data]
hf_outputs = hf_model.predict(text_pairs).tolist()
for i in range(len(vllm_outputs)):
assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01)
@pytest.mark.asyncio
async def test_rerank_api_texts(server: RemoteOpenAIServer):
query = "What is the capital of France?"
documents = [
"The capital of Brazil is Brasilia.",
"The capital of France is Paris.",
]
rerank_response = requests.post(
server.url_for("rerank"),
json={
"model": MODEL_NAME,
"query": query,
"documents": documents,
},
)
rerank_response.raise_for_status()
rerank = RerankResponse.model_validate(rerank_response.json())
assert rerank.id is not None
assert rerank.results is not None
assert len(rerank.results) == 2
paris_result = next(r for r in rerank.results if r.index == 1)
brazil_result = next(r for r in rerank.results if r.index == 0)
assert paris_result.relevance_score > brazil_result.relevance_score
@pytest.mark.asyncio
async def test_rerank_api_top_n(server: RemoteOpenAIServer):
query = "What is the capital of France?"
documents = [
"The capital of Brazil is Brasilia.",
"The capital of France is Paris.",
"Cross-encoder models are neat",
]
rerank_response = requests.post(
server.url_for("rerank"),
json={"model": MODEL_NAME, "query": query, "documents": documents, "top_n": 2},
)
rerank_response.raise_for_status()
rerank = RerankResponse.model_validate(rerank_response.json())
assert rerank.id is not None
assert rerank.results is not None
assert len(rerank.results) == 2
assert rerank.results[0].index == 1
@pytest.mark.asyncio
async def test_rerank_api_max_model_len(server: RemoteOpenAIServer):
query = "What is the capital of France?" * 100
documents = [
"The capital of Brazil is Brasilia.",
"The capital of France is Paris.",
]
rerank_response = requests.post(
server.url_for("rerank"),
json={"model": MODEL_NAME, "query": query, "documents": documents},
)
assert rerank_response.status_code == 400
# Assert just a small fragments of the response
assert "Please reduce the length of the input prompt" in rerank_response.text
@pytest.mark.asyncio
async def test_score_api_max_model_len(server: RemoteOpenAIServer):
queries = "What is the capital of France?" * 20
documents = [
"The capital of Brazil is Brasilia.",
"The capital of France is Paris.",
]
score_response = requests.post(
server.url_for("score"),
json={
"model": MODEL_NAME,
"queries": queries,
"documents": documents,
},
)
assert score_response.status_code == 400
# Assert just a small fragments of the response
assert "Please reduce the length of the input prompt" in score_response.text
# Test truncation
score_response = requests.post(
server.url_for("score"),
json={
"model": MODEL_NAME,
"queries": queries,
"documents": documents,
"truncate_prompt_tokens": 101,
},
)
assert score_response.status_code == 400
assert "Please request a smaller truncation size." in score_response.text
@pytest.mark.asyncio
async def test_invocations(server: RemoteOpenAIServer):
query = "What is the capital of France?"
documents = [
"The capital of Brazil is Brasilia.",
"The capital of France is Paris.",
]
request_args = {
"model": MODEL_NAME,
"query": query,
"documents": documents,
}
rerank_response = requests.post(server.url_for("rerank"), json=request_args)
rerank_response.raise_for_status()
invocation_response = requests.post(
server.url_for("invocations"), json=request_args
)
invocation_response.raise_for_status()
rerank_output = rerank_response.json()
invocation_output = invocation_response.json()
assert rerank_output.keys() == invocation_output.keys()
for rerank_result, invocations_result in zip(
rerank_output["results"], invocation_output["results"]
):
assert rerank_result.keys() == invocations_result.keys()
assert rerank_result["relevance_score"] == pytest.approx(
invocations_result["relevance_score"], rel=0.01
)
@pytest.mark.asyncio
async def test_pooling_embed(server: RemoteOpenAIServer):
response = requests.post(
server.url_for("pooling"),
json={
"model": MODEL_NAME,
"input": input_text,
"encoding_format": "float",
"task": "embed",
},
)
poolings = PoolingResponse.model_validate(response.json())
assert len(poolings.data) == 1
assert len(poolings.data[0].data) == EMBEDDING_SIZE
@pytest.mark.asyncio
@pytest.mark.parametrize("task", ["classify", "token_classify", "plugin"])
async def test_pooling_not_supported(server: RemoteOpenAIServer, task: str):
response = requests.post(
server.url_for("pooling"),
json={
"model": MODEL_NAME,
"input": input_text,
"encoding_format": "float",
"task": task,
},
)
assert response.json()["error"]["type"] == "BadRequestError"
if task == "plugin":
err_msg = "No IOProcessor plugin installed."
else:
err_msg = f"Unsupported task: {task!r}"
assert response.json()["error"]["message"].startswith(err_msg)
@@ -1,56 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
import pytest
from tests.models.language.pooling_mteb_test.mteb_score_utils import (
MTEB_RERANK_LANGS,
MTEB_RERANK_TASKS,
MTEB_RERANK_TOL,
RerankClientMtebEncoder,
ScoreClientMtebEncoder,
run_mteb_rerank,
)
from tests.utils import RemoteOpenAIServer
os.environ["VLLM_LOGGING_LEVEL"] = "WARNING"
MODEL_NAME = "cross-encoder/ms-marco-MiniLM-L-6-v2"
st_main_score = 0.33457
@pytest.fixture(scope="module")
def server():
args = ["--runner", "pooling", "--enforce-eager", "--disable-uvicorn-access-log"]
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
yield remote_server
def test_mteb_score(server):
url = server.url_for("score")
encoder = ScoreClientMtebEncoder(MODEL_NAME, url)
vllm_main_score = run_mteb_rerank(encoder, MTEB_RERANK_TASKS, MTEB_RERANK_LANGS)
print("VLLM main score: ", vllm_main_score)
print("SentenceTransformer main score: ", st_main_score)
print("Difference: ", st_main_score - vllm_main_score)
# We are not concerned that the vllm mteb results are better
# than SentenceTransformers, so we only perform one-sided testing.
assert st_main_score - vllm_main_score < MTEB_RERANK_TOL
def test_mteb_rerank(server):
url = server.url_for("rerank")
encoder = RerankClientMtebEncoder(MODEL_NAME, url)
vllm_main_score = run_mteb_rerank(encoder, MTEB_RERANK_TASKS, MTEB_RERANK_LANGS)
print("VLLM main score: ", vllm_main_score)
print("SentenceTransformer main score: ", st_main_score)
print("Difference: ", st_main_score - vllm_main_score)
# We are not concerned that the vllm mteb results are better
# than SentenceTransformers, so we only perform one-sided testing.
assert st_main_score - vllm_main_score < MTEB_RERANK_TOL
@@ -1,154 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import weakref
import pytest
import torch
from tests.models.utils import softmax
from vllm import LLM, PoolingParams
MODEL_NAME = "tomaarsen/Qwen3-Reranker-0.6B-seq-cls"
PROMPT = "The chef prepared a delicious meal."
TEXTS_1 = [
"What is the capital of France?",
"What is the capital of Germany?",
]
TEXTS_2 = [
"The capital of France is Paris.",
"The capital of Germany is Berlin.",
]
@pytest.fixture(scope="module")
def llm(vllm_runner):
with vllm_runner(
MODEL_NAME,
max_model_len=None,
max_num_batched_tokens=32768,
tensor_parallel_size=1,
gpu_memory_utilization=0.75,
enforce_eager=True,
seed=0,
enable_chunked_prefill=None,
) as runner:
# pytest caches yielded fixtures until after teardown, so use a proxy to
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
yield weakref.proxy(runner.llm)
@pytest.fixture(scope="module")
def hf_model(hf_runner):
return hf_runner(MODEL_NAME, is_cross_encoder=True)
@pytest.mark.skip_global_cleanup
def test_1_to_1(llm, hf_model):
text_pair = [TEXTS_1[0], TEXTS_2[0]]
hf_outputs = hf_model.predict([text_pair]).tolist()
vllm_outputs = [
output.outputs.score for output in llm.score(text_pair[0], text_pair[1])
]
assert len(vllm_outputs) == 1
assert len(hf_outputs) == 1
assert hf_outputs[0] == pytest.approx(vllm_outputs[0], rel=0.01)
@pytest.mark.skip_global_cleanup
def test_1_to_n(llm, hf_model):
text_pairs = [
[TEXTS_1[0], TEXTS_2[0]],
[TEXTS_1[0], TEXTS_2[1]],
]
vllm_outputs = [output.outputs.score for output in llm.score(TEXTS_1[0], TEXTS_2)]
hf_outputs = hf_model.predict(text_pairs).tolist()
assert len(vllm_outputs) == 2
assert len(hf_outputs) == 2
assert hf_outputs[0] == pytest.approx(vllm_outputs[0], rel=0.01)
assert hf_outputs[1] == pytest.approx(vllm_outputs[1], rel=0.01)
@pytest.mark.skip_global_cleanup
def test_n_to_n(llm, hf_model):
text_pairs = [
[TEXTS_1[0], TEXTS_2[0]],
[TEXTS_1[1], TEXTS_2[1]],
]
vllm_outputs = [output.outputs.score for output in llm.score(TEXTS_1, TEXTS_2)]
hf_outputs = hf_model.predict(text_pairs).tolist()
assert len(vllm_outputs) == 2
assert len(hf_outputs) == 2
assert hf_outputs[0] == pytest.approx(vllm_outputs[0], rel=0.01)
assert hf_outputs[1] == pytest.approx(vllm_outputs[1], rel=0.01)
@pytest.mark.skip_global_cleanup
def test_classify(llm):
outputs = llm.encode(PROMPT, pooling_task="classify", use_tqdm=False)
assert len(outputs) == 1
assert len(outputs[0].outputs.data) == 1
@pytest.mark.skip_global_cleanup
def test_max_tokens_per_doc(llm: LLM):
"""Test max_tokens_per_doc via PoolingParams.extra_kwargs (offline)."""
long_doc = "The capital of France is Paris. " * 20
# Without truncation
outputs_no_limit = llm.score(
TEXTS_1[0],
long_doc,
use_tqdm=False,
)
# With truncation via extra_kwargs
outputs_with_limit = llm.score(
TEXTS_1[0],
long_doc,
pooling_params=PoolingParams(extra_kwargs={"max_tokens_per_doc": 10}),
use_tqdm=False,
)
assert len(outputs_no_limit) == 1
assert len(outputs_with_limit) == 1
# Truncated version should have fewer prompt tokens
no_limit_tokens = len(outputs_no_limit[0].prompt_token_ids)
with_limit_tokens = len(outputs_with_limit[0].prompt_token_ids)
assert with_limit_tokens < no_limit_tokens
def test_pooling_params(llm: LLM):
def get_outputs(use_activation):
outputs = llm.score(
TEXTS_1[0],
TEXTS_2[0],
pooling_params=PoolingParams(use_activation=use_activation),
use_tqdm=False,
)
return torch.tensor([x.outputs.score for x in outputs])
default = get_outputs(use_activation=None)
w_activation = get_outputs(use_activation=True)
wo_activation = get_outputs(use_activation=False)
assert torch.allclose(default, w_activation, atol=1e-2), (
"Default should use activation."
)
assert not torch.allclose(w_activation, wo_activation, atol=1e-2), (
"wo_activation should not use activation."
)
assert torch.allclose(softmax(wo_activation), w_activation, atol=1e-2), (
"w_activation should be close to activation(wo_activation)."
)
@@ -1,512 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import requests
import torch
import torch.nn.functional as F
from tests.utils import RemoteOpenAIServer
from vllm.entrypoints.pooling.pooling.protocol import PoolingResponse
from vllm.entrypoints.pooling.scoring.protocol import RerankResponse, ScoreResponse
MODEL_NAME = "BAAI/bge-reranker-base"
DTYPE = "half"
input_text = "This product was excellent and exceeded my expectations"
input_tokens = [0, 3293, 12996, 509, 40881, 136, 204839, 297, 759, 202702, 2]
TEXTS_1 = [
"What is the capital of France?",
"What is the capital of Germany?",
]
TEXTS_2 = [
"The capital of France is Paris.",
"The capital of Germany is Berlin.",
]
@pytest.fixture(scope="module")
def server():
args = ["--enforce-eager", "--max-model-len", "100", "--dtype", DTYPE]
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
yield remote_server
@pytest.fixture(scope="module")
def hf_model(hf_runner):
return hf_runner(MODEL_NAME, is_cross_encoder=True)
@pytest.mark.asyncio
async def test_basic(server: RemoteOpenAIServer):
# test /v1/models
response = requests.get(server.url_for("/v1/models"))
served_model = response.json()["data"][0]["id"]
assert served_model == MODEL_NAME
# test /tokenize
response = requests.post(
server.url_for("/tokenize"),
json={"model": MODEL_NAME, "prompt": input_text},
)
assert response.json()["tokens"] == input_tokens
@pytest.mark.asyncio
async def test_score_api_queries_str_1_documents_str_1(
hf_model, server: RemoteOpenAIServer
):
score_response = requests.post(
server.url_for("score"),
json={
"model": MODEL_NAME,
"queries": TEXTS_1[0],
"documents": TEXTS_2[0],
},
)
score_response.raise_for_status()
score = ScoreResponse.model_validate(score_response.json())
assert score.id is not None
assert score.data is not None
assert len(score.data) == 1
vllm_outputs = [d.score for d in score.data]
hf_outputs = hf_model.predict([[TEXTS_1[0], TEXTS_2[0]]]).tolist()
for i in range(len(vllm_outputs)):
assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01)
@pytest.mark.asyncio
async def test_score_api_queries_str_1_documents_str_n(
hf_model, server: RemoteOpenAIServer
):
text_pairs = [
[TEXTS_1[0], TEXTS_2[0]],
[TEXTS_1[0], TEXTS_2[1]],
]
score_response = requests.post(
server.url_for("score"),
json={
"model": MODEL_NAME,
"queries": TEXTS_1[0],
"documents": TEXTS_2,
},
)
score_response.raise_for_status()
score = ScoreResponse.model_validate(score_response.json())
assert score.id is not None
assert score.data is not None
assert len(score.data) == 2
vllm_outputs = [d.score for d in score.data]
hf_outputs = hf_model.predict(text_pairs).tolist()
for i in range(len(vllm_outputs)):
assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01)
@pytest.mark.asyncio
async def test_score_api_queries_str_n_documents_str_n(
hf_model, server: RemoteOpenAIServer
):
text_pairs = [
[TEXTS_1[0], TEXTS_2[0]],
[TEXTS_1[1], TEXTS_2[1]],
]
score_response = requests.post(
server.url_for("score"),
json={
"model": MODEL_NAME,
"queries": TEXTS_1,
"documents": TEXTS_2,
},
)
score_response.raise_for_status()
score = ScoreResponse.model_validate(score_response.json())
assert score.id is not None
assert score.data is not None
assert len(score.data) == 2
vllm_outputs = [d.score for d in score.data]
hf_outputs = hf_model.predict(text_pairs).tolist()
for i in range(len(vllm_outputs)):
assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01)
@pytest.mark.asyncio
async def test_score_api_queries_vs_items(hf_model, server: RemoteOpenAIServer):
text_pairs = [
[TEXTS_1[0], TEXTS_2[0]],
[TEXTS_1[1], TEXTS_2[1]],
]
score_response = requests.post(
server.url_for("score"),
json={
"model": MODEL_NAME,
"queries": TEXTS_1,
"items": TEXTS_2,
},
)
score_response.raise_for_status()
score = ScoreResponse.model_validate(score_response.json())
assert score.id is not None
assert score.data is not None
assert len(score.data) == 2
vllm_outputs = [d.score for d in score.data]
hf_outputs = hf_model.predict(text_pairs).tolist()
for i in range(len(vllm_outputs)):
assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01)
@pytest.mark.asyncio
async def test_score_api_text_1_vs_text_2(hf_model, server: RemoteOpenAIServer):
text_pairs = [
[TEXTS_1[0], TEXTS_2[0]],
[TEXTS_1[1], TEXTS_2[1]],
]
score_response = requests.post(
server.url_for("score"),
json={
"model": MODEL_NAME,
"text_1": TEXTS_1,
"text_2": TEXTS_2,
},
)
score_response.raise_for_status()
score = ScoreResponse.model_validate(score_response.json())
assert score.id is not None
assert score.data is not None
assert len(score.data) == 2
vllm_outputs = [d.score for d in score.data]
hf_outputs = hf_model.predict(text_pairs).tolist()
for i in range(len(vllm_outputs)):
assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01)
@pytest.mark.asyncio
async def test_score_api_data_1_vs_data_2(hf_model, server: RemoteOpenAIServer):
text_pairs = [
[TEXTS_1[0], TEXTS_2[0]],
[TEXTS_1[1], TEXTS_2[1]],
]
score_response = requests.post(
server.url_for("score"),
json={
"model": MODEL_NAME,
"data_1": TEXTS_1,
"data_2": TEXTS_2,
},
)
score_response.raise_for_status()
score = ScoreResponse.model_validate(score_response.json())
assert score.id is not None
assert score.data is not None
assert len(score.data) == 2
vllm_outputs = [d.score for d in score.data]
hf_outputs = hf_model.predict(text_pairs).tolist()
for i in range(len(vllm_outputs)):
assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01)
@pytest.mark.asyncio
async def test_rerank_api_texts(server: RemoteOpenAIServer):
query = "What is the capital of France?"
documents = [
"The capital of Brazil is Brasilia.",
"The capital of France is Paris.",
]
rerank_response = requests.post(
server.url_for("rerank"),
json={
"model": MODEL_NAME,
"query": query,
"documents": documents,
},
)
rerank_response.raise_for_status()
rerank = RerankResponse.model_validate(rerank_response.json())
assert rerank.id is not None
assert rerank.results is not None
assert len(rerank.results) == 2
assert rerank.results[0].relevance_score >= 0.9
assert rerank.results[1].relevance_score <= 0.01
@pytest.mark.asyncio
async def test_rerank_api_top_n(server: RemoteOpenAIServer):
query = "What is the capital of France?"
documents = [
"The capital of Brazil is Brasilia.",
"The capital of France is Paris.",
"Cross-encoder models are neat",
]
rerank_response = requests.post(
server.url_for("rerank"),
json={"model": MODEL_NAME, "query": query, "documents": documents, "top_n": 2},
)
rerank_response.raise_for_status()
rerank = RerankResponse.model_validate(rerank_response.json())
assert rerank.id is not None
assert rerank.results is not None
assert len(rerank.results) == 2
assert rerank.results[0].relevance_score >= 0.9
assert rerank.results[1].relevance_score <= 0.01
@pytest.mark.asyncio
async def test_rerank_api_max_model_len(server: RemoteOpenAIServer):
query = "What is the capital of France?" * 100
documents = [
"The capital of Brazil is Brasilia.",
"The capital of France is Paris.",
]
rerank_response = requests.post(
server.url_for("rerank"),
json={"model": MODEL_NAME, "query": query, "documents": documents},
)
assert rerank_response.status_code == 400
# Assert just a small fragments of the response
assert "Please reduce the length of the input prompt" in rerank_response.text
@pytest.mark.asyncio
async def test_score_api_max_model_len(server: RemoteOpenAIServer):
queries = "What is the capital of France?" * 20
documents = [
"The capital of Brazil is Brasilia.",
"The capital of France is Paris.",
]
score_response = requests.post(
server.url_for("score"),
json={
"model": MODEL_NAME,
"queries": queries,
"documents": documents,
},
)
assert score_response.status_code == 400
# Assert just a small fragments of the response
assert "Please reduce the length of the input prompt" in score_response.text
# Test truncation
score_response = requests.post(
server.url_for("score"),
json={
"model": MODEL_NAME,
"queries": queries,
"documents": documents,
"truncate_prompt_tokens": 101,
},
)
assert score_response.status_code == 400
assert "Please request a smaller truncation size." in score_response.text
@pytest.mark.asyncio
async def test_invocations(server: RemoteOpenAIServer):
query = "What is the capital of France?"
documents = [
"The capital of Brazil is Brasilia.",
"The capital of France is Paris.",
]
request_args = {
"model": MODEL_NAME,
"query": query,
"documents": documents,
}
rerank_response = requests.post(server.url_for("rerank"), json=request_args)
rerank_response.raise_for_status()
invocation_response = requests.post(
server.url_for("invocations"), json=request_args
)
invocation_response.raise_for_status()
rerank_output = rerank_response.json()
invocation_output = invocation_response.json()
assert rerank_output.keys() == invocation_output.keys()
for rerank_result, invocations_result in zip(
rerank_output["results"], invocation_output["results"]
):
assert rerank_result.keys() == invocations_result.keys()
assert rerank_result["relevance_score"] == pytest.approx(
invocations_result["relevance_score"], rel=0.01
)
@pytest.mark.asyncio
async def test_use_activation(server: RemoteOpenAIServer):
async def get_outputs(use_activation):
query = "What is the capital of France?"
documents = [
"The capital of Brazil is Brasilia.",
"The capital of France is Paris.",
]
response = requests.post(
server.url_for("rerank"),
json={
"model": MODEL_NAME,
"query": query,
"documents": documents,
"use_activation": use_activation,
},
)
outputs = response.json()
return torch.tensor([x["relevance_score"] for x in outputs["results"]])
default = await get_outputs(use_activation=None)
w_activation = await get_outputs(use_activation=True)
wo_activation = await get_outputs(use_activation=False)
assert torch.allclose(default, w_activation, atol=1e-2), (
"Default should use activation."
)
assert not torch.allclose(w_activation, wo_activation, atol=1e-2), (
"wo_activation should not use activation."
)
assert torch.allclose(F.sigmoid(wo_activation), w_activation, atol=1e-2), (
"w_activation should be close to activation(wo_activation)."
)
@pytest.mark.asyncio
async def test_pooling_classify(server: RemoteOpenAIServer):
response = requests.post(
server.url_for("pooling"),
json={
"model": MODEL_NAME,
"input": input_text,
"encoding_format": "float",
"task": "classify",
},
)
poolings = PoolingResponse.model_validate(response.json())
assert len(poolings.data) == 1
assert len(poolings.data[0].data) == 1
@pytest.mark.asyncio
async def test_rerank_max_tokens_per_doc(
server: RemoteOpenAIServer,
):
"""Test that max_tokens_per_doc actually reduces the token count."""
query = "What is the capital of France?"
# Use a doc that fits within max_model_len=100 (query ~8 tokens + 4 special)
long_doc = "The capital of France is Paris. " * 10 # ~70 tokens
# Without max_tokens_per_doc
response_no_limit = requests.post(
server.url_for("rerank"),
json={
"model": MODEL_NAME,
"query": query,
"documents": [long_doc],
"truncate_prompt_tokens": 99,
},
)
response_no_limit.raise_for_status()
rerank_no_limit = RerankResponse.model_validate(response_no_limit.json())
# With max_tokens_per_doc
response_with_limit = requests.post(
server.url_for("rerank"),
json={
"model": MODEL_NAME,
"query": query,
"documents": [long_doc],
"max_tokens_per_doc": 10,
},
)
response_with_limit.raise_for_status()
rerank_with_limit = RerankResponse.model_validate(response_with_limit.json())
assert rerank_with_limit.usage.prompt_tokens < rerank_no_limit.usage.prompt_tokens
@pytest.mark.asyncio
async def test_rerank_max_tokens_per_doc_validation(
server: RemoteOpenAIServer,
):
"""Test that max_tokens_per_doc validation works correctly."""
query = "What is the capital of France?"
documents = ["The capital of France is Paris."]
# Test with max_tokens_per_doc=0 (should succeed — means no truncation)
response = requests.post(
server.url_for("rerank"),
json={
"model": MODEL_NAME,
"query": query,
"documents": documents,
"max_tokens_per_doc": 0,
},
)
response.raise_for_status()
# Test with invalid max_tokens_per_doc (negative)
response = requests.post(
server.url_for("rerank"),
json={
"model": MODEL_NAME,
"query": query,
"documents": documents,
"max_tokens_per_doc": -5,
},
)
assert response.status_code == 400
assert "max_tokens_per_doc must be a non-negative integer" in response.text
@pytest.mark.asyncio
@pytest.mark.parametrize("task", ["embed", "token_embed", "token_classify", "plugin"])
async def test_pooling_not_supported(server: RemoteOpenAIServer, task: str):
response = requests.post(
server.url_for("pooling"),
json={
"model": MODEL_NAME,
"input": input_text,
"encoding_format": "float",
"task": task,
},
)
assert response.json()["error"]["type"] == "BadRequestError"
if task == "plugin":
err_msg = "No IOProcessor plugin installed."
elif task == "token_classify":
err_msg = "Try switching the model's pooling_task via"
else:
err_msg = f"Unsupported task: {task!r}"
assert response.json()["error"]["message"].startswith(err_msg)
@@ -1,488 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import pytest
import requests
from tests.utils import VLLM_PATH, RemoteOpenAIServer
from vllm.entrypoints.pooling.scoring.protocol import RerankResponse, ScoreResponse
from vllm.multimodal.utils import encode_image_url, fetch_image
from vllm.platforms import current_platform
MODEL_NAME = "Qwen/Qwen3-VL-Reranker-2B"
HF_OVERRIDES = {
"architectures": ["Qwen3VLForSequenceClassification"],
"classifier_from_token": ["no", "yes"],
"is_original_qwen3_reranker": True,
}
ROCM_ATTN_BACKENDS = [
"ROCM_ATTN",
"ROCM_AITER_FA",
"TRITON_ATTN",
"FLEX_ATTENTION",
]
ATTN_BACKENDS = ROCM_ATTN_BACKENDS if current_platform.is_rocm() else ["auto"]
# The absolute floor matters only for the low ~0.10 text-vs-text score, where a
# few thousandths of drift would otherwise exceed the relative bound.
REL_TOL = 0.05
ABS_TOL = 0.007
# ROCm: disable prefix caching and eliminate batch variance to reduce
# test flakiness.
ROCM_EXTRA_ARGS = (
["--no-enable-prefix-caching", "--max-num-seqs", "1"]
if current_platform.is_rocm()
else []
)
def assert_score(actual: float, expected: float, backend: str, label: str):
tol = REL_TOL
abs_tol = ABS_TOL
diff = abs(actual - expected)
rel_diff = diff / abs(expected) if expected != 0 else diff
print(
f"[{backend}] {label}: actual={actual:.6f} expected={expected:.6f} "
f"diff={diff:.6f} rel_diff={rel_diff:.4f} tol={tol} abs_tol={abs_tol}"
)
assert actual == pytest.approx(expected, rel=tol, abs=abs_tol), (
f"[{backend}] {label}: score mismatch — "
f"actual={actual:.6f}, expected={expected:.6f}, "
f"rel_diff={rel_diff:.4f}, tol={tol}, abs_tol={abs_tol}"
)
query = "A cat standing in the snow."
document = "This product was excellent and exceeded my expectations."
image_url = "https://vllm-public-assets.s3.us-west-2.amazonaws.com/multimodal_asset/cat_snow.jpg"
documents = [
{
"type": "text",
"text": document,
},
{
"type": "image_url",
"image_url": {"url": image_url},
},
{
"type": "image_url",
"image_url": {"url": encode_image_url(fetch_image(image_url))},
},
]
# One bf16 ULP in attention compounds to ~0.1% here, so ROCm backends each
# settle differently; these are their midpoint across backends and archs.
if current_platform.is_rocm():
TEXT_VS_TEXT = 0.105243
TEXT_VS_IMAGE = 0.742472
TEXT_VS_TEXT_PLUS_IMAGE = 0.544568
else:
TEXT_VS_TEXT = 0.10040374100208282
TEXT_VS_IMAGE = 0.7423753142356873
TEXT_VS_TEXT_PLUS_IMAGE = 0.5298863053321838
@pytest.fixture(scope="module", params=ATTN_BACKENDS)
def server(request):
backend = request.param
print(f"\n=== Starting server with attention backend: {backend} ===")
args = [
"--enforce-eager",
"--max-model-len",
"8192",
"--chat-template",
str(VLLM_PATH / "examples/pooling/score/template/qwen3_vl_reranker.jinja"),
]
env = dict()
if backend != "auto":
args += ["--attention-config", json.dumps({"backend": backend})]
args += ROCM_EXTRA_ARGS
if backend != "ROCM_AITER_FA":
env["VLLM_ROCM_USE_AITER"] = "0"
with RemoteOpenAIServer(
MODEL_NAME, args, override_hf_configs=HF_OVERRIDES, env_dict=env
) as remote_server:
print(f"=== Server ready with backend: {backend} ===")
yield remote_server, backend
@pytest.mark.asyncio
async def test_score_api_queries_str_documents_str(
server: tuple[RemoteOpenAIServer, str],
):
remote_server, backend = server
score_response = requests.post(
remote_server.url_for("score"),
json={
"model": MODEL_NAME,
"queries": query,
"documents": document,
},
)
score_response.raise_for_status()
score = ScoreResponse.model_validate(score_response.json())
assert score.id is not None
assert score.data is not None
assert len(score.data) == 1
assert score.usage.prompt_tokens == 81
assert_score(score.data[0].score, TEXT_VS_TEXT, backend, "text_vs_text")
@pytest.mark.asyncio
async def test_score_api_queries_str_documents_text_content(
server: tuple[RemoteOpenAIServer, str],
):
remote_server, backend = server
score_response = requests.post(
remote_server.url_for("score"),
json={
"model": MODEL_NAME,
"queries": query,
"documents": {"content": [documents[0]]},
},
)
score_response.raise_for_status()
score = ScoreResponse.model_validate(score_response.json())
assert score.id is not None
assert score.data is not None
assert len(score.data) == 1
assert score.usage.prompt_tokens == 81
assert_score(score.data[0].score, TEXT_VS_TEXT, backend, "text_vs_text")
@pytest.mark.asyncio
async def test_score_api_queries_str_documents_image_url_content(
server: tuple[RemoteOpenAIServer, str],
):
remote_server, backend = server
score_response = requests.post(
remote_server.url_for("score"),
json={
"model": MODEL_NAME,
"queries": query,
"documents": {"content": [documents[1]]},
},
)
score_response.raise_for_status()
score = ScoreResponse.model_validate(score_response.json())
assert score.id is not None
assert score.data is not None
assert len(score.data) == 1
assert score.usage.prompt_tokens == 98
assert_score(score.data[0].score, TEXT_VS_IMAGE, backend, "text_vs_image")
@pytest.mark.asyncio
async def test_score_api_queries_str_documents_image_base64_content(
server: tuple[RemoteOpenAIServer, str],
):
remote_server, backend = server
score_response = requests.post(
remote_server.url_for("score"),
json={
"model": MODEL_NAME,
"queries": query,
"documents": {"content": [documents[2]]},
},
)
score_response.raise_for_status()
score = ScoreResponse.model_validate(score_response.json())
assert score.id is not None
assert score.data is not None
assert len(score.data) == 1
assert score.usage.prompt_tokens == 98
assert_score(score.data[0].score, TEXT_VS_IMAGE, backend, "text_vs_image_base64")
@pytest.mark.asyncio
async def test_score_api_queries_str_documents_image_url_plus_text_content(
server: tuple[RemoteOpenAIServer, str],
):
remote_server, backend = server
score_response = requests.post(
remote_server.url_for("score"),
json={
"model": MODEL_NAME,
"queries": query,
"documents": {"content": [documents[0], documents[1]]},
},
)
score_response.raise_for_status()
score = ScoreResponse.model_validate(score_response.json())
assert score.id is not None
assert score.data is not None
assert len(score.data) == 1
assert score.usage.prompt_tokens == 107
assert_score(
score.data[0].score, TEXT_VS_TEXT_PLUS_IMAGE, backend, "text_vs_text_plus_image"
)
@pytest.mark.asyncio
async def test_score_api_queries_str_documents_list(
server: tuple[RemoteOpenAIServer, str],
):
remote_server, backend = server
score_response = requests.post(
remote_server.url_for("score"),
json={
"model": MODEL_NAME,
"queries": query,
"documents": [
document,
{"content": [documents[0]]},
{"content": [documents[1]]},
{"content": [documents[0], documents[1]]},
],
},
)
score_response.raise_for_status()
score = ScoreResponse.model_validate(score_response.json())
assert score.id is not None
assert score.data is not None
assert len(score.data) == 4
assert score.usage.prompt_tokens == 367
assert_score(score.data[0].score, TEXT_VS_TEXT, backend, "list[0]_text_vs_text")
assert_score(score.data[1].score, TEXT_VS_TEXT, backend, "list[1]_text_vs_text")
assert_score(score.data[2].score, TEXT_VS_IMAGE, backend, "list[2]_text_vs_image")
assert_score(
score.data[3].score,
TEXT_VS_TEXT_PLUS_IMAGE,
backend,
"list[3]_text_vs_text_plus_image",
)
@pytest.mark.asyncio
async def test_rerank_api_queries_str_documents_list(
server: tuple[RemoteOpenAIServer, str],
):
remote_server, backend = server
rerank_response = requests.post(
remote_server.url_for("rerank"),
json={
"model": MODEL_NAME,
"query": query,
"documents": [
document,
{"content": [documents[0]]},
{"content": [documents[1]]},
{"content": [documents[0], documents[1]]},
],
},
)
rerank_response.raise_for_status()
rerank = RerankResponse.model_validate(rerank_response.json())
assert rerank.id is not None
assert rerank.model is not None
assert rerank.usage is not None
assert len(rerank.results) == 4
rerank.results.sort(key=lambda x: x.index)
assert_score(
rerank.results[0].relevance_score,
TEXT_VS_TEXT,
backend,
"rerank[0]_text_vs_text",
)
assert_score(
rerank.results[1].relevance_score,
TEXT_VS_TEXT,
backend,
"rerank[1]_text_vs_text",
)
assert_score(
rerank.results[2].relevance_score,
TEXT_VS_IMAGE,
backend,
"rerank[2]_text_vs_image",
)
assert_score(
rerank.results[3].relevance_score,
TEXT_VS_TEXT_PLUS_IMAGE,
backend,
"rerank[3]_text_vs_text_plus_image",
)
@pytest.mark.asyncio
async def test_score_api_queries_list_documents_list(
server: tuple[RemoteOpenAIServer, str],
):
remote_server, backend = server
score_response = requests.post(
remote_server.url_for("score"),
json={
"model": MODEL_NAME,
"queries": [query] * 4,
"documents": [
document,
{"content": [documents[0]]},
{"content": [documents[1]]},
{"content": [documents[0], documents[1]]},
],
},
)
score_response.raise_for_status()
score = ScoreResponse.model_validate(score_response.json())
assert score.id is not None
assert score.data is not None
assert len(score.data) == 4
assert score.usage.prompt_tokens == 367
assert_score(score.data[0].score, TEXT_VS_TEXT, backend, "paired[0]_text_vs_text")
assert_score(score.data[1].score, TEXT_VS_TEXT, backend, "paired[1]_text_vs_text")
assert_score(score.data[2].score, TEXT_VS_IMAGE, backend, "paired[2]_text_vs_image")
assert_score(
score.data[3].score,
TEXT_VS_TEXT_PLUS_IMAGE,
backend,
"paired[3]_text_vs_text_plus_image",
)
INSTRUCTION = (
"Given a multimodal retrieval query, retrieve candidates that "
"visually or textually match the requested scene, object, or action."
)
@pytest.mark.asyncio
async def test_score_api_instruction_field(
server: tuple[RemoteOpenAIServer, str],
):
remote_server, _ = server
default_response = requests.post(
remote_server.url_for("score"),
json={
"model": MODEL_NAME,
"queries": query,
"documents": document,
},
)
default_response.raise_for_status()
default_score = ScoreResponse.model_validate(default_response.json())
instruction_response = requests.post(
remote_server.url_for("score"),
json={
"model": MODEL_NAME,
"queries": query,
"documents": document,
"instruction": INSTRUCTION,
},
)
instruction_response.raise_for_status()
instruction_score = ScoreResponse.model_validate(instruction_response.json())
assert instruction_score.id is not None
assert instruction_score.data is not None
assert len(instruction_score.data) == 1
assert instruction_score.usage.prompt_tokens > default_score.usage.prompt_tokens
@pytest.mark.asyncio
async def test_rerank_api_instruction_field(
server: tuple[RemoteOpenAIServer, str],
):
remote_server, _ = server
doc_list = [
document,
{"content": [documents[0]]},
{"content": [documents[1]]},
{"content": [documents[0], documents[1]]},
]
default_response = requests.post(
remote_server.url_for("rerank"),
json={
"model": MODEL_NAME,
"query": query,
"documents": doc_list,
},
)
default_response.raise_for_status()
default_rerank = RerankResponse.model_validate(default_response.json())
instruction_response = requests.post(
remote_server.url_for("rerank"),
json={
"model": MODEL_NAME,
"query": query,
"documents": doc_list,
"instruction": INSTRUCTION,
},
)
instruction_response.raise_for_status()
instruction_rerank = RerankResponse.model_validate(instruction_response.json())
assert instruction_rerank.id is not None
assert instruction_rerank.model is not None
assert instruction_rerank.usage is not None
assert len(instruction_rerank.results) == len(default_rerank.results)
assert instruction_rerank.usage.prompt_tokens > default_rerank.usage.prompt_tokens
@pytest.mark.asyncio
async def test_rerank_api_instruction_field_matches_chat_template_kwargs(
server: tuple[RemoteOpenAIServer, str],
):
remote_server, backend = server
doc_list = [
document,
{"content": [documents[0]]},
{"content": [documents[1]]},
{"content": [documents[0], documents[1]]},
]
field_response = requests.post(
remote_server.url_for("rerank"),
json={
"model": MODEL_NAME,
"query": query,
"documents": doc_list,
"instruction": INSTRUCTION,
},
)
field_response.raise_for_status()
field_rerank = RerankResponse.model_validate(field_response.json())
kwargs_response = requests.post(
remote_server.url_for("rerank"),
json={
"model": MODEL_NAME,
"query": query,
"documents": doc_list,
"chat_template_kwargs": {"instruction": INSTRUCTION},
},
)
kwargs_response.raise_for_status()
kwargs_rerank = RerankResponse.model_validate(kwargs_response.json())
assert kwargs_rerank.usage.prompt_tokens == field_rerank.usage.prompt_tokens
field_scores = [
r.relevance_score for r in sorted(field_rerank.results, key=lambda x: x.index)
]
kwargs_scores = [
r.relevance_score for r in sorted(kwargs_rerank.results, key=lambda x: x.index)
]
assert field_scores == pytest.approx(kwargs_scores, rel=REL_TOL, abs=ABS_TOL)
@@ -1,247 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for ScoringIOProcessor helpers."""
from dataclasses import dataclass
from types import SimpleNamespace
import pytest
import torch
from tokenizers import Tokenizer, models, pre_tokenizers, processors
from transformers import TokenizersBackend
from vllm import PoolingParams, TokensPrompt
from vllm.entrypoints.pooling.scoring.io_processor import (
BiEncoderIOProcessor,
CrossEncoderIOProcessor,
JinaRankingIOProcessor,
LateInteractionIOProcessor,
_apply_post_tokenization_to_token_type_ids,
)
from vllm.entrypoints.pooling.scoring.typing import ScoringData
from vllm.entrypoints.pooling.scoring.utils import compress_token_type_ids
from vllm.entrypoints.pooling.typing import OfflineScoringInputsContext
from vllm.outputs import PoolingOutput, PoolingRequestOutput, RequestError
from vllm.renderers import TokenizeParams
pytestmark = pytest.mark.skip_global_cleanup
@dataclass
class _DummyTokenizer:
truncation_side: str = "left"
# Outside the range of the prompt ids below, so a test can tell a pad
# token apart from a real one.
pad_token_id: int = 99999
pad_token_type_id: int = 0
@pytest.fixture
def llm_reranker_processor() -> CrossEncoderIOProcessor:
vocab = {
"[UNK]": 0,
"[CLS]": 1,
"[SEP]": 2,
"a": 3,
"bc": 4,
"##b": 5,
"##c": 6,
}
backend = Tokenizer(models.WordPiece(vocab, unk_token="[UNK]"))
backend.pre_tokenizer = pre_tokenizers.WhitespaceSplit()
backend.post_processor = processors.TemplateProcessing(
single="[CLS] $A [SEP]",
special_tokens=[("[CLS]", 1), ("[SEP]", 2)],
)
processor = CrossEncoderIOProcessor.__new__(CrossEncoderIOProcessor)
processor.model_config = SimpleNamespace(enable_prompt_embeds=False)
processor.tokenizer = TokenizersBackend(
tokenizer_object=backend,
unk_token="[UNK]",
cls_token="[CLS]",
sep_token="[SEP]",
)
processor.supports_score_template = False
processor.use_sep_token = False
processor.model = None
return processor
def test_llm_reranker_tokenization_is_independent_of_nonbinding_doc_limit(
llm_reranker_processor: CrossEncoderIOProcessor,
):
encode_kwargs = {"add_special_tokens": True}
_, uncapped = llm_reranker_processor.get_score_prompt("a", "bc", encode_kwargs)
_, nonbinding = llm_reranker_processor.get_score_prompt(
"a", "bc", encode_kwargs, max_tokens_per_doc=10
)
expected = [1, 3, 5, 6, 2]
assert uncapped["prompt_token_ids"] == expected
assert nonbinding["prompt_token_ids"] == expected
@pytest.mark.parametrize(
"default_template,request_template,expected",
[
("{{ messages[0].content }} {{ messages[1].content }}", None, "a bc"),
(
"{{ messages[0].content }} {{ messages[1].content }}",
"{{ messages[1].content }} {{ messages[0].content }}",
"bc a",
),
(None, "{{ messages[1].content }} {{ messages[0].content }}", "bc a"),
(None, None, "abc"),
],
)
def test_offline_scoring_uses_default_template_unless_overridden(
llm_reranker_processor, default_template, request_template, expected
):
processor = llm_reranker_processor
processor.chat_template = default_template
processor.renderer = SimpleNamespace(
default_cmpl_tok_params=TokenizeParams(max_total_tokens=32)
)
ctx = OfflineScoringInputsContext(
pooling_task="classify",
tokenization_kwargs=None,
lora_request=None,
priorities=None,
scoring_data=ScoringData(data_1=["a"], data_2=["bc"]),
chat_template=request_template,
pooling_params=PoolingParams(),
)
factory, _ = processor.get_request_factory_offline(ctx)
(request,) = factory()
text, prompt = processor.get_score_prompt(
request["data_1"],
request["data_2"],
request["tok_params"].get_encode_kwargs(),
chat_template=request["chat_template"],
)
assert text == expected
assert prompt["prompt_token_ids"] == processor.tokenizer.encode(expected)
def test_token_type_ids_stay_aligned_with_a_truncated_padded_prompt():
"""The cross-encoder segment boundary must survive truncate + pad.
`token_type_ids` are parallel to `prompt_token_ids` and are reduced to the
document segment's start and end by `compress_token_type_ids`. If the two
arrays are truncated and padded in different orders they no longer
describe the same positions, and the model is told the query segment is
empty.
"""
tokenizer = _DummyTokenizer()
num_query, num_doc = 20, 30
prompt = TokensPrompt(prompt_token_ids=list(range(num_query + num_doc)))
token_type_ids = [0] * num_query + [1] * num_doc
tok_params = TokenizeParams(
max_total_tokens=100,
pad_prompt_tokens=-1,
truncate_prompt_tokens=40,
truncation_side="left",
)
processed_prompt = tok_params.apply_post_tokenization(
tokenizer,
prompt,
)
assert "prompt_token_ids" in processed_prompt
token_type_ids = _apply_post_tokenization_to_token_type_ids(
tokenizer, tok_params, token_type_ids
)
prompt_token_ids = processed_prompt["prompt_token_ids"]
assert len(token_type_ids) == len(prompt_token_ids)
# Keeping the last 40 tokens drops the first 10 query tokens, so 10 query
# tokens survive and the document starts at index 10.
doc_start, doc_end = compress_token_type_ids(token_type_ids)
assert (doc_start, doc_end) == (10, 40)
assert prompt_token_ids[:doc_start] == list(range(10, num_query))
assert prompt_token_ids[doc_start:doc_end] == list(range(num_query, 50))
assert token_type_ids[doc_end:] == [tokenizer.pad_token_type_id] * 60
def test_token_type_padding_uses_the_tokenizer_pad_type():
tokenizer = _DummyTokenizer()
tok_params = TokenizeParams(
max_total_tokens=8,
pad_prompt_tokens=8,
truncate_prompt_tokens=None,
truncation_side="right",
)
token_type_ids = _apply_post_tokenization_to_token_type_ids(
tokenizer, tok_params, [0, 0, 0, 1, 1]
)
assert token_type_ids == [0, 0, 0, 1, 1, 0, 0, 0]
assert compress_token_type_ids(token_type_ids) == (3, 5)
@pytest.mark.parametrize("token_type_ids", [[0, 1, 0, 1], [0, 2]])
def test_compress_token_type_ids_rejects_invalid_sequences(token_type_ids):
with pytest.raises(ValueError, match="optional trailing zeros"):
compress_token_type_ids(token_type_ids)
def _pooling_output(
request_id: str,
data: torch.Tensor,
error: RequestError | None = None,
) -> PoolingRequestOutput:
return PoolingRequestOutput(
request_id=request_id,
outputs=PoolingOutput(data),
prompt_token_ids=[1],
num_cached_tokens=0,
finished=True,
error=error,
)
@pytest.mark.parametrize(
"processor_type", [BiEncoderIOProcessor, LateInteractionIOProcessor]
)
@pytest.mark.parametrize("failed_index", [0, 1])
def test_paired_scoring_preserves_request_error(processor_type, failed_index):
processor = processor_type.__new__(processor_type)
processor.pad_token_id = None
error = RequestError(
code="multimodal_cache_miss",
message="Multi-modal processor cache miss.",
retryable=True,
)
outputs = [
_pooling_output("query", torch.tensor([1.0, 2.0])),
_pooling_output("document", torch.tensor([3.0, 4.0])),
]
outputs[failed_index] = _pooling_output(
outputs[failed_index].request_id,
torch.empty(0),
error,
)
(result,) = processor._post_process(outputs, n_queries=1)
assert result.request_id == "query_document"
assert result.error is error
assert result.outputs is outputs[failed_index].outputs
def test_jina_scoring_preserves_request_error_before_indexing_output():
processor = JinaRankingIOProcessor.__new__(JinaRankingIOProcessor)
error = RequestError(
code="multimodal_cache_miss",
message="Multi-modal processor cache miss.",
retryable=True,
)
failed = _pooling_output("request", torch.empty(0), error)
assert processor._post_process([failed], n_queries=1) == [failed]
@@ -1,150 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for JinaRankingIOProcessor request building."""
from types import SimpleNamespace
from unittest.mock import MagicMock
import pytest
from tokenizers import Tokenizer, models, pre_tokenizers
from transformers import TokenizersBackend
from vllm import PoolingParams
from vllm.entrypoints.pooling.base.io_processor import PoolingIOProcessor
from vllm.entrypoints.pooling.scoring.io_processor import JinaRankingIOProcessor
from vllm.entrypoints.pooling.scoring.protocol import RerankRequest
from vllm.entrypoints.pooling.scoring.typing import ScoringData
from vllm.entrypoints.pooling.typing import OfflineScoringInputsContext
from vllm.renderers import TokenizeParams
pytestmark = pytest.mark.skip_global_cleanup
def test_online_forwards_truncate_prompt_tokens_to_proxy(monkeypatch):
"""The proxy request handed to the base factory must carry
truncate_prompt_tokens/truncation_side from the real request.
JinaRankingIOProcessor swaps ctx.request for a proxy
PoolingCompletionRequest before delegating to the base factory, which
reads truncation off ctx.request. Dropping the fields on the proxy
silently disables truncate_prompt_tokens for Jina rerank/score.
"""
proc = JinaRankingIOProcessor.__new__(JinaRankingIOProcessor)
monkeypatch.setattr(
proc,
"valid_inputs_online",
MagicMock(return_value=ScoringData(data_1=["query"], data_2=["doc"])),
)
monkeypatch.setattr(proc, "_get_token_limits", MagicMock(return_value=(0, 0)))
monkeypatch.setattr(
proc, "ensure_str", MagicMock(side_effect=lambda data: list(data))
)
monkeypatch.setattr(
proc, "format_docs_prompts_func", MagicMock(return_value="formatted prompt")
)
captured: dict[str, object] = {}
def _spy_base(self, ctx):
captured["truncate_prompt_tokens"] = ctx.request.truncate_prompt_tokens
captured["truncation_side"] = ctx.request.truncation_side
return []
monkeypatch.setattr(PoolingIOProcessor, "get_request_factory_online", _spy_base)
request = RerankRequest(
model="m",
query="query",
documents=["doc"],
truncate_prompt_tokens=512,
truncation_side="left",
)
ctx = MagicMock()
ctx.request = request
ctx.prompt_extras = None
proc.get_request_factory_online(ctx)
assert captured["truncate_prompt_tokens"] == 512
assert captured["truncation_side"] == "left"
# The real request is restored after delegating.
assert ctx.request is request
@pytest.fixture
def offline_processor_and_context():
backend = Tokenizer(models.WordLevel({"[UNK]": 0}, unk_token="[UNK]"))
backend.pre_tokenizer = pre_tokenizers.WhitespaceSplit()
proc = JinaRankingIOProcessor.__new__(JinaRankingIOProcessor)
proc.tokenizer = TokenizersBackend(tokenizer_object=backend)
proc.model_config = SimpleNamespace(max_model_len=1024, is_encoder_decoder=False)
proc.renderer = SimpleNamespace(
default_cmpl_tok_params=TokenizeParams(max_total_tokens=1024)
)
ctx = OfflineScoringInputsContext(
pooling_task="token_embed",
scoring_data=ScoringData(
data_1=["query alpha beta", "query gamma delta"],
data_2=["document one two", "document three four"],
),
pooling_params=PoolingParams(
extra_kwargs={
"chat_template_kwargs": {"instruction": "Keep this instruction"}
}
),
tokenization_kwargs=None,
chat_template=None,
lora_request=None,
priorities=None,
)
return proc, ctx
@pytest.mark.parametrize(
("query_limit", "doc_limit", "n_queries"),
[(None, None, 1), (0, 0, 2), (1, None, 1), (None, 2, 1), (1, 2, 1), (1, 2, 2)],
ids=["unset", "zero", "query", "doc", "both-1-to-n", "both-n-to-n"],
)
def test_offline_truncates_text_before_formatting(
offline_processor_and_context, query_limit, doc_limit, n_queries
):
proc, ctx = offline_processor_and_context
queries = ctx.scoring_data.data_1[:n_queries]
docs = ctx.scoring_data.data_2
ctx.scoring_data = ScoringData(data_1=queries, data_2=docs)
ctx.pooling_params.extra_kwargs.update(
{
name: limit
for name, limit in (
("max_tokens_per_query", query_limit),
("max_tokens_per_doc", doc_limit),
)
if limit is not None
}
)
factory, num_requests = proc.get_request_factory_offline(ctx)
requests = list(factory())
assert len(requests) == num_requests == n_queries
for i, request in enumerate(requests):
prompt = request["prompts"]["prompt"]
query = " ".join(queries[i].split()[: query_limit or None])
assert f"<query>\n{query}<|rerank_token|>\n</query>" in prompt
prompt_docs = docs if n_queries == 1 else [docs[i]]
for j, doc in enumerate(prompt_docs):
doc = " ".join(doc.split()[: doc_limit or None])
assert f'<passage id="{j}">\n{doc}<|embed_token|>\n</passage>' in prompt
assert prompt.count("<|embed_token|>") == len(prompt_docs)
assert prompt.count("<|rerank_token|>") == 1
assert "<instruct>\nKeep this instruction\n</instruct>" in prompt
@pytest.mark.parametrize("name", ["max_tokens_per_query", "max_tokens_per_doc"])
@pytest.mark.parametrize("limit", [-1, 1024])
def test_offline_rejects_invalid_token_limits(
offline_processor_and_context, name, limit
):
proc, ctx = offline_processor_and_context
ctx.pooling_params.extra_kwargs[name] = limit
with pytest.raises(ValueError, match=name):
proc.get_request_factory_offline(ctx)
@@ -1,105 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import weakref
import pytest
from .util import ColBERTScoringHfRunner
MODEL_NAME = "answerdotai/answerai-colbert-small-v1"
COLBERT_DIM = 96
LINEAR_WEIGHTS_KEY = "linear.weight"
PROMPT = "The chef prepared a delicious meal."
TEXTS_1 = [
"What is the capital of France?",
"What is the capital of Germany?",
]
TEXTS_2 = [
"The capital of France is Paris.",
"The capital of Germany is Berlin.",
]
DTYPE = "half"
@pytest.fixture(scope="module")
def llm(vllm_runner):
with vllm_runner(
MODEL_NAME,
max_model_len=None,
max_num_batched_tokens=32768,
tensor_parallel_size=1,
gpu_memory_utilization=0.75,
enforce_eager=True,
seed=0,
enable_chunked_prefill=None,
) as runner:
# pytest caches yielded fixtures until after teardown, so use a proxy to
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
yield weakref.proxy(runner.llm)
@pytest.fixture(scope="module")
def hf_model():
return ColBERTScoringHfRunner(
model_name=MODEL_NAME, linear_weights_key=LINEAR_WEIGHTS_KEY
)
@pytest.mark.skip_global_cleanup
def test_1_to_1(llm, hf_model):
text_pair = [TEXTS_1[0], TEXTS_2[0]]
hf_outputs = hf_model.predict([text_pair]).tolist()
vllm_outputs = [
output.outputs.score for output in llm.score(text_pair[0], text_pair[1])
]
assert len(vllm_outputs) == 1
assert len(hf_outputs) == 1
assert hf_outputs[0] == pytest.approx(vllm_outputs[0], rel=0.01)
@pytest.mark.skip_global_cleanup
def test_1_to_n(llm, hf_model):
text_pairs = [
[TEXTS_1[0], TEXTS_2[0]],
[TEXTS_1[0], TEXTS_2[1]],
]
hf_outputs = hf_model.predict(text_pairs).tolist()
vllm_outputs = [output.outputs.score for output in llm.score(TEXTS_1[0], TEXTS_2)]
assert len(vllm_outputs) == 2
assert len(hf_outputs) == 2
assert hf_outputs[0] == pytest.approx(vllm_outputs[0], rel=0.01)
assert hf_outputs[1] == pytest.approx(vllm_outputs[1], rel=0.01)
@pytest.mark.skip_global_cleanup
def test_n_to_n(llm, hf_model):
text_pairs = [
[TEXTS_1[0], TEXTS_2[0]],
[TEXTS_1[1], TEXTS_2[1]],
]
hf_outputs = hf_model.predict(text_pairs).tolist()
vllm_outputs = [output.outputs.score for output in llm.score(TEXTS_1, TEXTS_2)]
assert len(vllm_outputs) == 2
assert len(hf_outputs) == 2
assert hf_outputs[0] == pytest.approx(vllm_outputs[0], rel=0.01)
assert hf_outputs[1] == pytest.approx(vllm_outputs[1], rel=0.01)
def test_token_embed(llm):
outputs = llm.encode(PROMPT, pooling_task="token_embed", use_tqdm=False)
assert len(outputs) == 1
assert outputs[0].outputs.data.shape == (9, COLBERT_DIM)
@@ -1,79 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import weakref
import pytest
from .util import make_base64_image, make_image_mm_param
MODEL_NAME = "vidore/colpali-v1.3-hf"
@pytest.fixture(scope="module")
def llm(vllm_runner):
with vllm_runner(
MODEL_NAME,
max_model_len=None,
max_num_batched_tokens=32768,
tensor_parallel_size=1,
gpu_memory_utilization=0.75,
enforce_eager=True,
seed=0,
enable_chunked_prefill=None,
) as runner:
# pytest caches yielded fixtures until after teardown, so use a proxy to
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
yield weakref.proxy(runner.llm)
@pytest.mark.skip_global_cleanup
def test_query_text_vs_docs_image(llm):
"""Score a text query against image documents via the multimodal path."""
red_image = make_base64_image(64, 64, color=(255, 0, 0))
blue_image = make_base64_image(64, 64, color=(0, 0, 255))
query = "Describe the red object"
image_docs = [
make_image_mm_param(red_image),
make_image_mm_param(blue_image),
]
scores = llm.score(query, image_docs)
assert len(scores) == 2
assert scores[0].outputs.score > scores[1].outputs.score
@pytest.mark.skip_global_cleanup
def test_query_text_vs_docs_mix(llm) -> None:
"""Score a text query against a mix of text and image documents."""
red_image = make_base64_image(64, 64, color=(255, 0, 0))
query = "What is the capital of France?"
documents: list = [
"The capital of France is Paris.",
make_image_mm_param(red_image),
]
scores = llm.score(query, documents)
assert len(scores) == 2
assert scores[0].outputs.score > scores[1].outputs.score
@pytest.mark.skip_global_cleanup
def test_query_image_vs_docs_text(llm) -> None:
"""Score an image query against text documents."""
red_image = make_base64_image(64, 64, color=(255, 0, 0))
image_query = make_image_mm_param(red_image, text="red color")
documents = [
"Describe the red object.",
"The capital of France is Paris.",
]
scores = llm.score(image_query, documents)
assert len(scores) == 2
assert scores[0].outputs.score > scores[1].outputs.score
@@ -1,267 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Online API tests for ColBERT late interaction scoring."""
import pytest
import requests
from tests.utils import RemoteOpenAIServer
from vllm.entrypoints.pooling.scoring.protocol import RerankResponse, ScoreResponse
from .util import ColBERTScoringHfRunner
MODEL_NAME = "answerdotai/answerai-colbert-small-v1"
COLBERT_DIM = 96
MAX_MODEL_LEN = 512
LINEAR_WEIGHTS_KEY = "linear.weight"
TEXTS_1 = [
"What is the capital of France?",
"What is the capital of Germany?",
]
TEXTS_2 = [
"The capital of France is Paris.",
"The capital of Germany is Berlin.",
]
@pytest.mark.asyncio
async def test_flash_late_interaction_rejects_mm_processor_kwargs_by_default():
from unittest.mock import MagicMock
from vllm.config.multimodal import MultiModalConfig
from vllm.entrypoints.pooling.base.serving import PoolingBaseServing
from vllm.entrypoints.pooling.scoring.protocol import ScoreQueriesDocumentsRequest
from vllm.entrypoints.pooling.scoring.serving import ServingScores
from vllm.exceptions import VLLMValidationError
serving = MagicMock()
serving.model_config = MagicMock(multimodal_config=MultiModalConfig())
serving.trust_request_mm_kwargs = False
serving.io_processor = MagicMock()
serving._init_ctx = PoolingBaseServing._init_ctx.__get__(serving)
request = ScoreQueriesDocumentsRequest(
model=MODEL_NAME,
queries=TEXTS_1[0],
documents=TEXTS_2[0],
mm_processor_kwargs={"vision_min_num_patches": 3_000_000_000},
)
with pytest.raises(
VLLMValidationError,
match="Per-request mm_processor_kwargs are disabled",
):
await ServingScores.flash_late_interaction(serving, request)
serving._preprocessing.assert_not_called()
@pytest.fixture(scope="module", params=[True, False])
def server(request):
args = [
"--max-model-len",
str(MAX_MODEL_LEN),
]
# Test run pooling score MaxSim on worker side (GPU)
# aka flash-late-interaction
if not request.param:
args += ["--no-enable-flash-late-interaction"]
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
yield remote_server
@pytest.fixture(scope="module")
def hf_model():
return ColBERTScoringHfRunner(
model_name=MODEL_NAME, linear_weights_key=LINEAR_WEIGHTS_KEY
)
@pytest.mark.asyncio
async def test_score_api_queries_str_1_documents_str_1(
hf_model, server: RemoteOpenAIServer
):
score_response = requests.post(
server.url_for("score"),
json={
"model": MODEL_NAME,
"queries": TEXTS_1[0],
"documents": TEXTS_2[0],
},
)
score_response.raise_for_status()
score = ScoreResponse.model_validate(score_response.json())
assert score.id is not None
assert score.data is not None
assert len(score.data) == 1
vllm_outputs = [d.score for d in score.data]
hf_outputs = hf_model.predict([[TEXTS_1[0], TEXTS_2[0]]]).tolist()
for i in range(len(vllm_outputs)):
assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01)
@pytest.mark.asyncio
async def test_score_api_queries_str_1_documents_str_n(
hf_model, server: RemoteOpenAIServer
):
text_pairs = [
[TEXTS_1[0], TEXTS_2[0]],
[TEXTS_1[0], TEXTS_2[1]],
]
score_response = requests.post(
server.url_for("score"),
json={
"model": MODEL_NAME,
"queries": TEXTS_1[0],
"documents": TEXTS_2,
},
)
score_response.raise_for_status()
score = ScoreResponse.model_validate(score_response.json())
assert score.id is not None
assert score.data is not None
assert len(score.data) == 2
vllm_outputs = [d.score for d in score.data]
hf_outputs = hf_model.predict(text_pairs).tolist()
for i in range(len(vllm_outputs)):
assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01)
@pytest.mark.asyncio
async def test_score_api_queries_str_n_documents_str_n(
hf_model, server: RemoteOpenAIServer
):
text_pairs = [
[TEXTS_1[0], TEXTS_2[0]],
[TEXTS_1[1], TEXTS_2[1]],
]
score_response = requests.post(
server.url_for("score"),
json={
"model": MODEL_NAME,
"queries": TEXTS_1,
"documents": TEXTS_2,
},
)
score_response.raise_for_status()
score = ScoreResponse.model_validate(score_response.json())
assert score.id is not None
assert score.data is not None
assert len(score.data) == 2
vllm_outputs = [d.score for d in score.data]
hf_outputs = hf_model.predict(text_pairs).tolist()
for i in range(len(vllm_outputs)):
assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01)
@pytest.mark.asyncio
async def test_rerank_api_texts(server: RemoteOpenAIServer):
"""Test ColBERT rerank endpoint."""
query = "What is the capital of France?"
documents = [
"The capital of Brazil is Brasilia.",
"The capital of France is Paris.",
]
rerank_response = requests.post(
server.url_for("rerank"),
json={
"model": MODEL_NAME,
"query": query,
"documents": documents,
},
)
rerank_response.raise_for_status()
rerank = RerankResponse.model_validate(rerank_response.json())
assert rerank.id is not None
assert rerank.results is not None
assert len(rerank.results) == 2
paris_result = next(r for r in rerank.results if r.index == 1)
brazil_result = next(r for r in rerank.results if r.index == 0)
assert paris_result.relevance_score > brazil_result.relevance_score
@pytest.mark.asyncio
async def test_rerank_api_top_n(server: RemoteOpenAIServer):
"""Test ColBERT rerank with top_n parameter."""
query = "What is the capital of France?"
documents = [
"The capital of Brazil is Brasilia.",
"The capital of France is Paris.",
"Machine learning is a field of AI.",
]
rerank_response = requests.post(
server.url_for("rerank"),
json={
"model": MODEL_NAME,
"query": query,
"documents": documents,
"top_n": 2,
},
)
rerank_response.raise_for_status()
rerank = RerankResponse.model_validate(rerank_response.json())
assert len(rerank.results) == 2
assert rerank.results[0].index == 1
@pytest.mark.asyncio
async def test_token_embed(server: RemoteOpenAIServer):
"""Test ColBERT token_embed task via pooling endpoint."""
text = "What is the capital of France?"
pooling_response = requests.post(
server.url_for("pooling"),
json={
"model": MODEL_NAME,
"input": text,
"task": "token_embed",
},
)
pooling_response.raise_for_status()
pooling = pooling_response.json()
assert "data" in pooling
assert len(pooling["data"]) == 1
embeddings = pooling["data"][0]["data"]
assert isinstance(embeddings, list)
assert len(embeddings) > 0
assert len(embeddings[0]) == COLBERT_DIM
@pytest.mark.asyncio
async def test_embed_not_supported(server: RemoteOpenAIServer):
"""Test that ColBERT model does not support 'embed' task."""
task = "embed"
text = "What is the capital of France?"
response = requests.post(
server.url_for("pooling"),
json={
"model": MODEL_NAME,
"input": text,
"task": task,
},
)
assert response.json()["error"]["type"] == "BadRequestError"
assert response.json()["error"]["message"].startswith(f"Unsupported task: {task!r}")
@@ -1,193 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import requests
from tests.entrypoints.pooling.scoring.util import (
make_base64_image,
make_image_mm_param,
)
from tests.utils import RemoteOpenAIServer
from vllm.entrypoints.pooling.scoring.protocol import RerankResponse, ScoreResponse
MODEL_NAME = "vidore/colpali-v1.3-hf"
@pytest.fixture(scope="module")
def server():
with RemoteOpenAIServer(MODEL_NAME, []) as remote_server:
yield remote_server
@pytest.mark.asyncio
async def test_score_api_query_text_vs_docs_image(server: RemoteOpenAIServer):
query = "Describe the red object"
red_image = make_base64_image(64, 64, color=(255, 0, 0))
blue_image = make_base64_image(64, 64, color=(0, 0, 255))
documents = [
make_image_mm_param(red_image),
make_image_mm_param(blue_image),
]
score_response = requests.post(
server.url_for("score"),
json={
"model": MODEL_NAME,
"queries": query,
"documents": documents,
},
)
score_response.raise_for_status()
scores = ScoreResponse.model_validate(score_response.json())
assert scores.id is not None
assert scores.data is not None
assert len(scores.data) == 2
assert scores.data[0].score > scores.data[1].score
@pytest.mark.asyncio
async def test_score_api_query_text_vs_docs_mix(server: RemoteOpenAIServer):
red_image = make_base64_image(64, 64, color=(255, 0, 0))
query = "What is the capital of France?"
documents: list = [
"The capital of France is Paris.",
make_image_mm_param(red_image),
]
score_response = requests.post(
server.url_for("score"),
json={
"model": MODEL_NAME,
"queries": query,
"documents": documents,
},
)
score_response.raise_for_status()
scores = ScoreResponse.model_validate(score_response.json())
assert scores.id is not None
assert scores.data is not None
assert len(scores.data) == 2
assert scores.data[0].score > scores.data[1].score
@pytest.mark.asyncio
async def test_score_api_query_image_vs_docs_text(server: RemoteOpenAIServer):
red_image = make_base64_image(64, 64, color=(255, 0, 0))
image_query = make_image_mm_param(red_image, text="red color")
documents = [
"Describe the red object.",
"The capital of France is Paris.",
]
score_response = requests.post(
server.url_for("score"),
json={
"model": MODEL_NAME,
"queries": image_query,
"documents": documents,
},
)
score_response.raise_for_status()
scores = ScoreResponse.model_validate(score_response.json())
assert scores.id is not None
assert scores.data is not None
assert len(scores.data) == 2
assert scores.data[0].score > scores.data[1].score
@pytest.mark.asyncio
async def test_rerank_api_query_text_vs_docs_image(server: RemoteOpenAIServer):
query = "Describe the red object"
red_image = make_base64_image(64, 64, color=(255, 0, 0))
blue_image = make_base64_image(64, 64, color=(0, 0, 255))
documents = [
make_image_mm_param(red_image),
make_image_mm_param(blue_image),
]
rerank_response = requests.post(
server.url_for("rerank"),
json={"model": MODEL_NAME, "query": query, "documents": documents},
)
rerank_response.raise_for_status()
rerank = RerankResponse.model_validate(rerank_response.json())
assert rerank.id is not None
assert rerank.results is not None
assert len(rerank.results) == 2
red_result = next(r for r in rerank.results if r.index == 0)
blue_result = next(r for r in rerank.results if r.index == 1)
assert red_result.relevance_score > blue_result.relevance_score
@pytest.mark.asyncio
async def test_rerank_api_query_text_vs_docs_mix(server: RemoteOpenAIServer):
red_image = make_base64_image(64, 64, color=(255, 0, 0))
query = "What is the capital of France?"
documents: list = [
"The capital of France is Paris.",
make_image_mm_param(red_image),
]
rerank_response = requests.post(
server.url_for("rerank"),
json={
"model": MODEL_NAME,
"query": query,
"documents": documents,
},
)
rerank_response.raise_for_status()
rerank = RerankResponse.model_validate(rerank_response.json())
assert rerank.id is not None
assert rerank.results is not None
assert len(rerank.results) == 2
result0 = next(r for r in rerank.results if r.index == 0)
result1 = next(r for r in rerank.results if r.index == 1)
assert result0.relevance_score > result1.relevance_score
@pytest.mark.asyncio
async def test_rerank_api_query_image_vs_docs_text(server: RemoteOpenAIServer):
red_image = make_base64_image(64, 64, color=(255, 0, 0))
image_query = make_image_mm_param(red_image, text="red color")
documents = [
"Describe the red object.",
"The capital of France is Paris.",
]
rerank_response = requests.post(
server.url_for("rerank"),
json={
"model": MODEL_NAME,
"query": image_query,
"documents": documents,
},
)
rerank_response.raise_for_status()
rerank = RerankResponse.model_validate(rerank_response.json())
assert rerank.id is not None
assert rerank.results is not None
assert len(rerank.results) == 2
result0 = next(r for r in rerank.results if r.index == 0)
result1 = next(r for r in rerank.results if r.index == 1)
assert result0.relevance_score > result1.relevance_score
@@ -1,141 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from unittest.mock import AsyncMock, Mock
import pytest
from vllm.entrypoints.pooling.scoring.serving import ServingScores
from vllm.entrypoints.pooling.typing import PoolingEngineInput, PoolingServeContext
from vllm.inputs import tokens_input
from vllm.pooling_params import PoolingParams
def _make_context() -> PoolingServeContext:
return PoolingServeContext(
request=Mock(),
model_name="model",
request_id="score-caller-controlled-id",
pooling_params=PoolingParams(task="token_embed"),
lora_request=None,
priorities=None,
prompt_extras=None,
engine_inputs=[
PoolingEngineInput(
prompts=tokens_input(prompt_token_ids=[1]),
params=PoolingParams(task="token_embed"),
lora_requests=None,
priorities=0,
),
PoolingEngineInput(
prompts=tokens_input(prompt_token_ids=[2]),
params=PoolingParams(task="token_embed"),
lora_requests=None,
priorities=0,
),
],
n_queries=1,
)
def _query_key(context: PoolingServeContext) -> str:
assert context.engine_inputs is not None
pooling_params = context.engine_inputs[0]["params"]
late_interaction_params = pooling_params.late_interaction_params
assert late_interaction_params is not None
query_key = late_interaction_params.query_key
assert query_key is not None
return query_key
@pytest.mark.asyncio
@pytest.mark.skip_global_cleanup
@pytest.mark.parametrize("failed_stage", ["query", "doc"])
async def test_flash_late_interaction_cleans_up_queries_on_failure(
failed_stage: str,
monkeypatch: pytest.MonkeyPatch,
):
serving = object.__new__(ServingScores)
serving.io_processor = Mock()
serving.engine_client = Mock(abort=AsyncMock(), collective_rpc=AsyncMock())
context = _make_context()
monkeypatch.setattr(serving, "_init_ctx", AsyncMock(return_value=context))
monkeypatch.setattr(serving, "_preprocessing", AsyncMock())
async def encode_queries(ctx):
ctx.late_interaction_query_keys = ["query-key"]
if failed_stage == "query":
raise RuntimeError("query failed")
async def encode_docs(ctx):
ctx.late_interaction_doc_keys = ["doc-key"]
if failed_stage == "doc":
raise RuntimeError("doc failed")
monkeypatch.setattr(
serving,
"_flash_late_interaction_encode_queries",
AsyncMock(side_effect=encode_queries),
)
monkeypatch.setattr(
serving,
"_flash_late_interaction_encode_docs",
AsyncMock(side_effect=encode_docs),
)
with pytest.raises(RuntimeError, match=f"{failed_stage} failed"):
await serving.flash_late_interaction()
expected_doc_keys = [] if failed_stage == "query" else ["doc-key"]
serving.engine_client.abort.assert_awaited_once_with(
["query-key", *expected_doc_keys]
)
serving.engine_client.collective_rpc.assert_awaited_once_with(
"release_late_interaction_query_cache",
args=(["query-key"],),
)
@pytest.mark.asyncio
async def test_colliding_request_ids_use_distinct_late_interaction_keys(
monkeypatch: pytest.MonkeyPatch,
):
serving = object.__new__(ServingScores)
serving.engine_client = Mock(abort=AsyncMock(), collective_rpc=AsyncMock())
prepare_generators = AsyncMock()
monkeypatch.setattr(serving, "_prepare_generators", prepare_generators)
monkeypatch.setattr(serving, "_collect_batch", AsyncMock())
first = _make_context()
second = _make_context()
for context in (first, second):
await serving._flash_late_interaction_encode_queries(context)
await serving._flash_late_interaction_encode_docs(context)
prepared_contexts = [call.args[0] for call in prepare_generators.await_args_list]
first_query_key = _query_key(prepared_contexts[0])
first_doc_key = _query_key(prepared_contexts[1])
second_query_key = _query_key(prepared_contexts[2])
second_doc_key = _query_key(prepared_contexts[3])
first_doc_request_ids = prepared_contexts[1].prompt_request_ids
second_doc_request_ids = prepared_contexts[3].prompt_request_ids
assert first_query_key == first_doc_key
assert second_query_key == second_doc_key
assert first_query_key != second_query_key
assert "caller-controlled-id" not in first_query_key
assert "caller-controlled-id" not in second_query_key
assert first_doc_request_ids is not None
assert second_doc_request_ids is not None
assert first.late_interaction_doc_keys == first_doc_request_ids
assert second.late_interaction_doc_keys == second_doc_request_ids
assert first_doc_request_ids != second_doc_request_ids
assert all("caller-controlled-id" not in key for key in first_doc_request_ids)
assert all("caller-controlled-id" not in key for key in second_doc_request_ids)
assert first.late_interaction_query_keys is not None
await serving._cleanup_flash_late_interaction(first)
serving.engine_client.abort.assert_awaited_once_with(
[*first.late_interaction_query_keys, *first_doc_request_ids]
)
@@ -1,84 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from tokenizers import Tokenizer, models, pre_tokenizers
from transformers import TokenizersBackend
from vllm.entrypoints.pooling.scoring.typing import ScoreInput
from vllm.entrypoints.pooling.scoring.utils import (
truncate_text_to_tokens,
validate_score_input,
)
from vllm.exceptions import VLLMValidationError
@pytest.mark.parametrize(
("data_1", "data_2", "is_multimodal_model", "architecture", "message"),
[
pytest.param(
{"content": []},
"document",
False,
"TestModel",
"MultiModalParam is not supported for TestModel",
id="unsupported-multimodal-input",
),
pytest.param(
["query 1", "query 2"],
["document"],
False,
"TestModel",
"Input lengths must be either 1:1, 1:N or N:N",
id="incompatible-input-lengths",
),
pytest.param(
[],
["document"],
False,
"TestModel",
"At least one text element must be given",
id="empty-first-input",
),
pytest.param(
["query"],
[],
False,
"TestModel",
"At least one text_pair element must be given",
id="empty-second-input",
),
],
)
def test_validate_score_input_rejects_invalid_inputs(
data_1: ScoreInput | list[ScoreInput],
data_2: ScoreInput | list[ScoreInput],
is_multimodal_model: bool,
architecture: str,
message: str,
):
with pytest.raises(VLLMValidationError) as exc_info:
validate_score_input(
data_1,
data_2,
is_multimodal_model=is_multimodal_model,
architecture=architecture,
)
assert str(exc_info.value) == message
assert exc_info.value.parameter is None
assert exc_info.value.value is None
def test_truncate_text_to_tokens_handles_shared_character_offsets():
vocab = {
token: i for i, token in enumerate(sorted(pre_tokenizers.ByteLevel.alphabet()))
}
backend = Tokenizer(models.BPE(vocab=vocab, merges=[]))
backend.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False)
tokenizer = TokenizersBackend(tokenizer_object=backend)
truncated = truncate_text_to_tokens("aéx", tokenizer, max_tokens=2)
assert truncated == "a"
assert len(tokenizer(truncated, add_special_tokens=False)["input_ids"]) == 1
-107
View File
@@ -1,107 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from io import BytesIO
import pybase64 as base64
import torch
import torch.nn.functional as F
from PIL import Image
from safetensors.torch import load_file
from transformers import AutoModel, AutoTokenizer
from tests.conftest import HfRunner
from vllm.entrypoints.pooling.scoring.typing import ScoreMultiModalParam
from vllm.entrypoints.pooling.scoring.utils import compute_maxsim_score
from vllm.renderers.chat_utils import (
ChatCompletionContentPartImageParam,
ChatCompletionContentPartTextParam,
)
from vllm.transformers_utils.repo_utils import hf_api
class ColBERTScoringHfRunner(torch.nn.Module):
def __init__(self, model_name, linear_weights_key):
super().__init__()
self.tokenizer = AutoTokenizer.from_pretrained(model_name)
self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
extra = {}
if self.device.type == "cpu":
extra["attn_implementation"] = "eager"
self.model = AutoModel.from_pretrained(
model_name,
**extra,
).to(self.device)
self.model.eval()
path = hf_api().hf_hub_download(model_name, filename="model.safetensors")
weights = load_file(path)
self.linear_weight = weights[linear_weights_key].to(self.device).float()
@torch.inference_mode()
def forward(self, texts):
embeddings = []
for text in texts:
inputs = self.tokenizer(text, return_tensors="pt").to(self.device)
hidden = self.model(**inputs).last_hidden_state.float()
projected = F.linear(hidden, self.linear_weight.float())
normalised = F.normalize(projected, p=2, dim=-1)
embeddings.append(normalised.squeeze(0).cpu())
return embeddings
@torch.inference_mode()
def predict(self, prompts: list[list[str]], *args, **kwargs):
hf_embeddings = [self(prompt) for prompt in prompts]
hf_outputs = [
compute_maxsim_score(*map(torch.tensor, pair)).item()
for pair in hf_embeddings
]
return torch.as_tensor(hf_outputs)
class EncoderScoringHfRunner(HfRunner):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs, is_sentence_transformer=True)
@torch.inference_mode()
def predict(self, prompts: list[list[str]], *args, **kwargs):
hf_embeddings = [self.encode(prompt) for prompt in prompts]
hf_outputs = [
F.cosine_similarity(*map(torch.tensor, pair), dim=0)
for pair in hf_embeddings
]
return torch.as_tensor(hf_outputs)
def make_base64_image(
width: int = 64, height: int = 64, color: tuple[int, int, int] = (255, 0, 0)
) -> str:
"""Create a small solid-color PNG image and return its base64 data URI."""
img = Image.new("RGB", (width, height), color)
buf = BytesIO()
img.save(buf, format="PNG")
b64 = base64.b64encode(buf.getvalue()).decode()
return f"data:image/png;base64,{b64}"
def make_image_mm_param(
image_uri: str,
text: str | None = None,
) -> ScoreMultiModalParam:
"""Build a ScoreMultiModalParam containing an image (and optional text)."""
content: list = [
ChatCompletionContentPartImageParam(
type="image_url",
image_url={"url": image_uri},
),
]
if text is not None:
content.append(
ChatCompletionContentPartTextParam(type="text", text=text),
)
return ScoreMultiModalParam(content=content)
@@ -1,94 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from unittest.mock import MagicMock
import pytest
from vllm.entrypoints.pooling import factories
from vllm.entrypoints.pooling.factories import init_pooling_io_processors
from vllm.entrypoints.pooling.pooling.io_processor import (
PluginWithIOProcessorPlugins,
UnsupportedCombinedTaskIOProcessor,
)
def _bge_m3_config(io_processor_plugin=None):
model_config = MagicMock()
model_config.get_pooling_task.return_value = "embed&token_classify"
model_config.io_processor_plugin = io_processor_plugin
model_config.hf_config.to_dict.return_value = {}
model_config.architecture = "BgeM3EmbeddingModel"
vllm_config = MagicMock(model_config=model_config)
renderer = MagicMock()
renderer._executor = MagicMock()
chat_template_config = MagicMock(
chat_template=None,
chat_template_content_format="auto",
trust_request_chat_template=False,
)
return vllm_config, renderer, chat_template_config
def test_combined_task_without_plugin_uses_rejection_processor():
vllm_config, renderer, chat_template_config = _bge_m3_config()
processors = init_pooling_io_processors(
supported_tasks=("embed", "embed&token_classify"),
vllm_config=vllm_config,
renderer=renderer,
chat_template_config=chat_template_config,
)
assert processors.keys() == {"embed&token_classify"}
assert isinstance(
processors["embed&token_classify"], UnsupportedCombinedTaskIOProcessor
)
def test_combined_task_with_plugin_uses_plugin_processor(monkeypatch):
vllm_config, renderer, chat_template_config = _bge_m3_config("bge_m3_sparse_plugin")
monkeypatch.setattr(factories, "has_io_processor", lambda *_: True)
monkeypatch.setattr(
"vllm.entrypoints.pooling.pooling.io_processor.get_io_processor",
lambda *_: MagicMock(),
)
processors = init_pooling_io_processors(
supported_tasks=("embed", "embed&token_classify"),
vllm_config=vllm_config,
renderer=renderer,
chat_template_config=chat_template_config,
)
assert processors.keys() == {"embed&token_classify", "plugin"}
assert isinstance(processors["plugin"], PluginWithIOProcessorPlugins)
def test_combined_task_plain_pooling_request_has_actionable_error(monkeypatch):
from vllm.entrypoints.pooling.pooling.protocol import PoolingCompletionRequest
from vllm.entrypoints.pooling.pooling.serving import ServingPooling
vllm_config, renderer, chat_template_config = _bge_m3_config("bge_m3_sparse_plugin")
monkeypatch.setattr(factories, "has_io_processor", lambda *_: True)
monkeypatch.setattr(
"vllm.entrypoints.pooling.pooling.io_processor.get_io_processor",
lambda *_: MagicMock(),
)
engine_client = MagicMock(renderer=renderer, vllm_config=vllm_config)
models = MagicMock(model_config=vllm_config.model_config)
serving = ServingPooling(
engine_client,
models,
supported_tasks=("embed", "embed&token_classify"),
request_logger=None,
chat_template_config=chat_template_config,
)
request = PoolingCompletionRequest(model="BAAI/bge-m3", input=["hola"])
assert serving.io_processors.keys() == {"embed&token_classify", "plugin"}
io_processor = serving.get_io_processor(request)
with pytest.raises(ValueError, match="plugin request with a 'data' field"):
io_processor.create_pooling_params(request)
@@ -1,260 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
from vllm import PoolingParams
from vllm.entrypoints.pooling.base.io_processor import PoolingIOProcessor
from vllm.entrypoints.pooling.offline import PoolingOfflineMixin
from vllm.entrypoints.pooling.scoring.io_processor import CrossEncoderIOProcessor
from vllm.entrypoints.pooling.typing import OfflineEncodeInputsContext
from vllm.exceptions import VLLMValidationError
from vllm.renderers import TokenizeParams
@pytest.fixture
def processor() -> PoolingIOProcessor:
return object.__new__(PoolingIOProcessor)
def test_rejects_untrusted_request_chat_template(processor: PoolingIOProcessor):
with pytest.raises(VLLMValidationError) as exc_info:
processor._validate_chat_template("template", None, False)
assert str(exc_info.value) == (
"Chat template is passed with request, but "
"--trust-request-chat-template is not set. "
"Refused request with untrusted chat template."
)
assert exc_info.value.parameter is None
assert exc_info.value.value is None
def test_rejects_mismatched_pooling_params(processor: PoolingIOProcessor):
with pytest.raises(VLLMValidationError) as exc_info:
processor._params_to_seq([PoolingParams()], num_requests=2)
assert str(exc_info.value) == (
"The lengths of prompts (2) and params (1) must be the same."
)
assert exc_info.value.parameter is None
assert exc_info.value.value is None
def test_rejects_mismatched_lora_requests(processor: PoolingIOProcessor):
with pytest.raises(VLLMValidationError) as exc_info:
processor._lora_request_to_seq([None], num_requests=2)
assert str(exc_info.value) == (
"The lengths of prompts (2) and lora_request (1) must be the same."
)
assert exc_info.value.parameter is None
assert exc_info.value.value is None
def test_rejects_conflicting_pooling_task(processor: PoolingIOProcessor):
processor.model_config = SimpleNamespace(is_encoder_decoder=False)
processor.renderer = SimpleNamespace(
default_cmpl_tok_params=TokenizeParams(max_total_tokens=None)
)
ctx = OfflineEncodeInputsContext(
pooling_task="embed",
tokenization_kwargs=None,
lora_request=None,
priorities=None,
prompts=[[1]],
pooling_params=PoolingParams(task="classify"),
)
with pytest.raises(VLLMValidationError) as exc_info:
processor.get_request_factory_offline(ctx)
assert str(exc_info.value) == (
"You cannot overwrite param.task='classify' with pooling_task='embed'!"
)
assert exc_info.value.parameter is None
assert exc_info.value.value is None
def test_qwen3_reranker_warns_without_chat_template(monkeypatch):
from unittest.mock import MagicMock
from vllm.entrypoints.pooling.scoring.io_processor import CrossEncoderIOProcessor
monkeypatch.setattr(
"vllm.model_executor.model_loader.get_model_cls", lambda *_: MagicMock()
)
monkeypatch.setattr(
"vllm.model_executor.models.interfaces.supports_score_template",
lambda *_: False,
)
monkeypatch.setattr(
"vllm.entrypoints.pooling.scoring.io_processor.is_mistral_tokenizer",
lambda *_: False,
)
mock_logger = MagicMock()
monkeypatch.setattr(
"vllm.entrypoints.pooling.scoring.io_processor.logger",
mock_logger,
)
model_config = MagicMock()
model_config.architecture = "Qwen3ForSequenceClassification"
model_config.model = "Qwen/Qwen3-Reranker-0.6B"
model_config.hf_config.is_original_qwen3_reranker = True
model_config.use_sep_token = False
model_config.is_multimodal_model = False
model_config.max_model_len = 8192
vllm_config = MagicMock(model_config=model_config)
tokenizer = MagicMock()
tokenizer.pad_token_id = 0
renderer = MagicMock()
renderer.get_tokenizer.return_value = tokenizer
chat_template_config = MagicMock(chat_template=None)
CrossEncoderIOProcessor(
vllm_config=vllm_config,
renderer=renderer,
chat_template_config=chat_template_config,
)
mock_logger.warning.assert_called_once()
warning_call = mock_logger.warning.call_args
assert "Qwen/Qwen3-Reranker-0.6B" in warning_call[0][1]
assert "qwen3_reranker.jinja" in warning_call[0][2]
def test_qwen3_vl_reranker_warns_with_vl_template(monkeypatch):
from unittest.mock import MagicMock
from vllm.entrypoints.pooling.scoring.io_processor import CrossEncoderIOProcessor
monkeypatch.setattr(
"vllm.model_executor.model_loader.get_model_cls", lambda *_: MagicMock()
)
monkeypatch.setattr(
"vllm.model_executor.models.interfaces.supports_score_template",
lambda *_: False,
)
monkeypatch.setattr(
"vllm.entrypoints.pooling.scoring.io_processor.is_mistral_tokenizer",
lambda *_: False,
)
mock_logger = MagicMock()
monkeypatch.setattr(
"vllm.entrypoints.pooling.scoring.io_processor.logger",
mock_logger,
)
model_config = MagicMock()
model_config.architecture = "Qwen3VLForSequenceClassification"
model_config.model = "Qwen/Qwen3-VL-Reranker-2B"
model_config.hf_config.is_original_qwen3_reranker = True
model_config.use_sep_token = False
model_config.is_multimodal_model = True
model_config.max_model_len = 8192
vllm_config = MagicMock(model_config=model_config)
tokenizer = MagicMock()
tokenizer.pad_token_id = 0
renderer = MagicMock()
renderer.get_tokenizer.return_value = tokenizer
chat_template_config = MagicMock(chat_template=None)
CrossEncoderIOProcessor(
vllm_config=vllm_config,
renderer=renderer,
chat_template_config=chat_template_config,
)
mock_logger.warning.assert_called_once()
warning_call = mock_logger.warning.call_args
assert "Qwen/Qwen3-VL-Reranker-2B" in warning_call[0][1]
assert "qwen3_vl_reranker.jinja" in warning_call[0][2]
def test_qwen3_reranker_no_warning_when_template_provided(monkeypatch):
from unittest.mock import MagicMock
from vllm.entrypoints.pooling.scoring.io_processor import CrossEncoderIOProcessor
monkeypatch.setattr(
"vllm.model_executor.model_loader.get_model_cls", lambda *_: MagicMock()
)
monkeypatch.setattr(
"vllm.model_executor.models.interfaces.supports_score_template",
lambda *_: False,
)
monkeypatch.setattr(
"vllm.entrypoints.pooling.scoring.io_processor.is_mistral_tokenizer",
lambda *_: False,
)
mock_logger = MagicMock()
monkeypatch.setattr(
"vllm.entrypoints.pooling.scoring.io_processor.logger",
mock_logger,
)
model_config = MagicMock()
model_config.architecture = "Qwen3ForSequenceClassification"
model_config.model = "Qwen/Qwen3-Reranker-0.6B"
model_config.hf_config.is_original_qwen3_reranker = True
model_config.use_sep_token = False
model_config.is_multimodal_model = False
model_config.max_model_len = 8192
vllm_config = MagicMock(model_config=model_config)
tokenizer = MagicMock()
tokenizer.pad_token_id = 0
renderer = MagicMock()
renderer.get_tokenizer.return_value = tokenizer
chat_template_config = MagicMock(chat_template="custom template")
CrossEncoderIOProcessor(
vllm_config=vllm_config,
renderer=renderer,
chat_template_config=chat_template_config,
)
mock_logger.warning.assert_not_called()
def _make_score_llm(monkeypatch) -> PoolingOfflineMixin:
"""A scoring LLM with no model behind it.
Input validation runs for real; only the rendering, engine and
post-processing boundaries are stubbed out. The pooling task is filled in
before any of them, so the mutation is observable without inference.
"""
proc = CrossEncoderIOProcessor.__new__(CrossEncoderIOProcessor)
proc.is_multimodal_model = False
proc.architecture = "CrossEncoder"
monkeypatch.setattr(proc, "get_request_factory_offline", lambda ctx: (None, 0))
monkeypatch.setattr(proc, "post_process_offline", lambda ctx: [])
llm = PoolingOfflineMixin.__new__(PoolingOfflineMixin)
llm.runner_type = "pooling"
llm.pooling_task = "classify" # SCORE_TYPE_MAP -> "cross-encoder"
llm.model_config = SimpleNamespace(hf_config=SimpleNamespace(num_labels=1))
llm.pooling_io_processors = {"cross-encoder": proc}
monkeypatch.setattr(llm, "_run_tiling_engine", lambda *args, **kwargs: [])
return llm
def test_score_leaves_caller_pooling_params_untouched(monkeypatch):
llm = _make_score_llm(monkeypatch)
params = PoolingParams()
llm.score("query", "doc", pooling_params=params)
assert params.task is None, (
"LLM.score() wrote the model's own pooling task into the caller's "
"PoolingParams; reusing that object with another pooling model now "
"fails with 'You cannot overwrite ...' before any inference runs."
)
@@ -1,314 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
from unittest import mock
from unittest.mock import Mock
import pytest
import torch
from vllm import PoolingParams
from vllm.config import PoolerConfig
from vllm.entrypoints.pooling.embed.io_processor import (
JinaRankingTokenEmbedIOProcessor,
TokenEmbedIOProcessor,
)
from vllm.entrypoints.pooling.late_chunking import build_late_chunking_metadata
from vllm.entrypoints.pooling.offline import PoolingOfflineMixin
from vllm.entrypoints.pooling.typing import (
OfflineEncodeInputsContext,
OfflineOutputsContext,
)
from vllm.exceptions import VLLMValidationError
from vllm.outputs import (
LateChunk,
PoolingOutput,
PoolingRequestOutput,
RequestError,
)
from vllm.pooling_params import LateChunkingParams
from vllm.renderers import TokenizeParams
@pytest.fixture
def late_chunk_processor():
processor = object.__new__(TokenEmbedIOProcessor)
processor.model_config = SimpleNamespace(
is_encoder_decoder=False,
architecture="NomicBertModel",
model_impl="auto",
is_matryoshka=False,
hf_config=SimpleNamespace(),
pooler_config=PoolerConfig(seq_pooling_type="MEAN", tok_pooling_type="ALL"),
)
processor.vllm_config = SimpleNamespace(
model_config=processor.model_config,
cache_config=SimpleNamespace(enable_prefix_caching=False),
scheduler_config=SimpleNamespace(enable_chunked_prefill=False),
lora_config=None,
)
processor.renderer = Mock()
processor.renderer.default_cmpl_tok_params = TokenizeParams(max_total_tokens=32)
processor.renderer.render_cmpl.side_effect = lambda **_: [
{
"type": "token",
"prompt_token_ids": [101, 1, 2, 102],
"prompt_token_offsets": [(0, 0), (0, 1), (2, 3), (0, 0)],
}
]
return processor
def _late_chunk_context(prompts="a b", **kwargs):
return OfflineEncodeInputsContext(
pooling_task="token_embed",
tokenization_kwargs=kwargs or None,
lora_request=None,
priorities=None,
prompts=prompts,
pooling_params=PoolingParams(
late_chunking_params=LateChunkingParams(chunk_size=2)
),
)
def test_late_chunk_render_requests_offsets_once(late_chunk_processor):
processor = late_chunk_processor
ctx = _late_chunk_context()
factory, count = processor.get_request_factory_offline(ctx)
render_params = next(factory())
result = processor.render(render_params)
assert count == 1
processor.renderer.render_cmpl.assert_called_once()
assert processor.renderer.render_cmpl.call_args.kwargs[
"tok_params"
].return_token_offsets
assert not render_params["tok_params"].return_token_offsets
assert "prompt_token_offsets" not in result["prompts"]
assert "late_chunking" not in result
assert [c.char_range for c in ctx.late_chunking[0].metadata.chunks] == [
(0, 1),
(2, 3),
]
assert ctx.late_chunking[0].metadata.input_tokens == 4
@pytest.mark.parametrize("as_sequence", [False, True])
@pytest.mark.parametrize("chunk_size", [None, 2])
def test_jina_ranking_rejects_chunking_before_tokenization(
late_chunk_processor, as_sequence, chunk_size
):
processor = object.__new__(JinaRankingTokenEmbedIOProcessor)
processor.__dict__.update(late_chunk_processor.__dict__)
ctx = _late_chunk_context(prompts=["document", "query"])
params = PoolingParams(
late_chunking_params=LateChunkingParams(chunk_size) if chunk_size else None
)
ctx.pooling_params = [params] if as_sequence else params
factory, count = processor.get_request_factory_offline(ctx)
assert count == 1
request = next(factory())
if chunk_size is not None:
with pytest.raises(VLLMValidationError, match="JinaForRanking.*late chunking"):
processor.render(request)
late_chunk_processor.renderer.render_cmpl.assert_not_called()
else:
result = processor.render(request)
assert result["prompts"]["type"] == "token"
assert result["prompts"]["prompt_token_ids"] == [101, 1, 2, 102]
late_chunk_processor.renderer.render_cmpl.assert_called_once()
assert not request["tok_params"].return_token_offsets
@pytest.mark.parametrize(
"prompts, kwargs, error",
[
pytest.param("", {}, "plain-text", id="empty"),
pytest.param([1, 2], {}, "plain-text", id="token-ids"),
pytest.param(
{"prompt": "abc", "multi_modal_data": {}},
{},
"plain-text",
id="multimodal",
),
]
+ [
pytest.param("abc", {key: value}, "does not support", id=key)
for key, value in (
("truncate_prompt_tokens", 2),
("truncation", True),
("pad_prompt_tokens", 8),
("padding", True),
("do_lower_case", True),
)
],
)
def test_late_chunking_rejects_unsupported_input_before_render(
late_chunk_processor, prompts, kwargs, error
):
processor = late_chunk_processor
with pytest.raises(VLLMValidationError, match=error):
factory, _ = processor.get_request_factory_offline(
_late_chunk_context(prompts, **kwargs)
)
processor.render(next(factory()))
processor.renderer.render_cmpl.assert_not_called()
def test_late_chunk_ranges_keep_unicode_overlaps_and_special_only_chunks():
text = "中 😀 e\u0301"
offsets = [(0, 0), (0, 1), (2, 3), (2, 3), (4, 6), (0, 0)]
metadata = build_late_chunking_metadata(text, len(offsets), offsets, 1)
assert [c.char_range for c in metadata.chunks] == [
None,
(0, 1),
(2, 3),
(2, 3),
(4, 6),
None,
]
assert [
text[slice(*c.char_range)] if c.char_range else None for c in metadata.chunks
] == [None, "中", "😀", "😀", "e\u0301", None]
chunks = build_late_chunking_metadata(text, len(offsets), offsets, 4).chunks
assert [(c.token_range, c.char_range) for c in chunks] == [
((0, 4), (0, 3)),
((4, 6), (4, 6)),
]
@pytest.mark.parametrize(
"offsets", [None, [(0, 1)], [(-1, 1), (0, 0)], [(0, 4), (0, 0)], [(2, 3), (0, 1)]]
)
def test_late_chunk_ranges_reject_missing_or_invalid_offsets(offsets):
with pytest.raises(VLLMValidationError, match="offsets"):
build_late_chunking_metadata("abc", 2, offsets, 2)
def _mock_chunk_llm(processor, outputs):
llm = mock.Mock(spec=PoolingOfflineMixin)
llm.encode = PoolingOfflineMixin.encode.__get__(llm)
llm._run_tiling_engine = PoolingOfflineMixin._run_tiling_engine.__get__(llm)
llm.pooling_io_processors = {"token_embed": processor}
llm._executor = SimpleNamespace(map=map)
llm.llm_engine = mock.Mock()
llm.llm_engine.vllm_config.scheduler_config.max_num_seqs = 2
llm.llm_engine.has_unfinished_requests.return_value = False
llm.llm_engine.step.side_effect = outputs
llm._render_and_add_requests = mock.Mock(
side_effect=lambda **kw: [
str(i) + "-internal" for i in range(len(kw["params"]))
]
)
return llm
def _chunk_output(request_id, rows=2, **kwargs):
return PoolingRequestOutput(
str(request_id),
PoolingOutput(torch.ones(rows, 4)),
[101, 1, 2, 102],
0,
True,
**kwargs,
)
def test_late_chunk_mapping_preserves_order_mixed_requests_and_errors(
late_chunk_processor,
):
error = RequestError("test_error", "original error")
failed = _chunk_output(3, rows=0, error=error)
llm = _mock_chunk_llm(late_chunk_processor, RuntimeError("step failed"))
params = [
PoolingParams(late_chunking_params=LateChunkingParams(2)),
PoolingParams(),
PoolingParams(late_chunking_params=LateChunkingParams(1)),
PoolingParams(late_chunking_params=LateChunkingParams(2)),
]
# A failed encode call must not leave metadata for the following batch.
with pytest.raises(RuntimeError, match="step failed"):
llm.encode(
["a b"] * 2,
pooling_task="token_embed",
pooling_params=params[:2],
use_tqdm=False,
)
llm.llm_engine.step.side_effect = [
[
_chunk_output(2, rows=4),
failed,
_chunk_output(1, rows=4),
_chunk_output(0),
]
]
outputs = llm.encode(
["a b"] * 4, pooling_task="token_embed", pooling_params=params, use_tqdm=False
)
assert outputs[0].late_chunking.chunks == [
LateChunk((0, 2), (0, 1)),
LateChunk((2, 4), (2, 3)),
]
assert outputs[1].late_chunking is None
assert len(outputs[2].late_chunking.chunks) == 4
assert outputs[3].late_chunking is None
assert outputs[3].error is error
def test_late_chunk_mapping_rejects_successful_output_with_wrong_row_count(
late_chunk_processor,
):
llm = _mock_chunk_llm(late_chunk_processor, [[_chunk_output(0, rows=0)]])
with pytest.raises(ValueError, match="does not match"):
llm.encode(
"a b",
pooling_task="token_embed",
pooling_params=PoolingParams(late_chunking_params=LateChunkingParams(2)),
use_tqdm=False,
)
# All engine requests have finished before post-processing validates the output.
llm.llm_engine.abort_request.assert_not_called()
def test_late_chunk_contexts_remain_isolated_when_rendering_interleaves(
late_chunk_processor,
):
processor = late_chunk_processor
first, second = _late_chunk_context("xy z"), _late_chunk_context()
shared = PoolingParams(late_chunking_params=LateChunkingParams(2))
first.pooling_params = second.pooling_params = shared
first_factory, _ = processor.get_request_factory_offline(first)
second_factory, _ = processor.get_request_factory_offline(second)
first_request, second_request = next(first_factory()), next(second_factory())
processor.render(second_request)
assert first.late_chunking[0].metadata is None
processor.renderer.render_cmpl.side_effect = lambda **_: [
{
"type": "token",
"prompt_token_ids": [101, 1, 2, 102],
"prompt_token_offsets": [(0, 0), (0, 2), (3, 4), (0, 0)],
}
]
processor.render(first_request)
first_output = processor.post_process_offline(
OfflineOutputsContext([_chunk_output(0)], late_chunking=first.late_chunking)
)[0]
second_output = processor.post_process_offline(
OfflineOutputsContext([_chunk_output(0)], late_chunking=second.late_chunking)
)[0]
assert first_output.late_chunking.chunks[0].char_range == (0, 2)
assert second_output.late_chunking.chunks[0].char_range == (0, 1)
def test_plain_token_embed_skips_chunk_metadata(late_chunk_processor):
ctx = _late_chunk_context()
ctx.pooling_params = PoolingParams()
factory, _ = late_chunk_processor.get_request_factory_offline(ctx)
request = next(factory())
result = late_chunk_processor.render(request)
assert not ctx.late_chunking
assert result["params"].late_chunking_params is None
assert not late_chunk_processor.renderer.render_cmpl.call_args.kwargs[
"tok_params"
].return_token_offsets
-59
View File
@@ -1,59 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import importlib
import importlib.util
import json
import warnings
from types import SimpleNamespace
import numpy as np
import pytest
import torch
from vllm.entrypoints.pooling.utils import encode_pooling_output_float_or_ndarray
def _pooling_output(data):
return SimpleNamespace(outputs=SimpleNamespace(data=data))
def test_encode_pooling_output_float_or_ndarray_returns_numpy_array():
output = _pooling_output(torch.tensor([1.0, 2.0, 3.0], dtype=torch.float32))
encoded = encode_pooling_output_float_or_ndarray(output)
assert isinstance(encoded, np.ndarray)
np.testing.assert_allclose(encoded, [1.0, 2.0, 3.0])
@pytest.mark.skipif(
importlib.util.find_spec("orjson") is None,
reason="orjson is not installed",
)
def test_orjson_serializes_numpy_array():
from fastapi.responses import ORJSONResponse
output = _pooling_output(torch.tensor([1.0, 2.0, 3.0], dtype=torch.float32))
encoded = encode_pooling_output_float_or_ndarray(output)
with warnings.catch_warnings():
warnings.simplefilter("ignore", DeprecationWarning)
response = ORJSONResponse(content={"embedding": encoded})
assert json.loads(response.body)["embedding"] == pytest.approx([1.0, 2.0, 3.0])
def test_encode_pooling_output_float_or_ndarray_falls_back_to_list():
class DataWithUnsupportedNumpy:
def is_contiguous(self):
return True
def numpy(self):
raise TypeError("unsupported dtype")
def tolist(self):
return [1.0, 2.0, 3.0]
output = _pooling_output(DataWithUnsupportedNumpy())
assert encode_pooling_output_float_or_ndarray(output) == [1.0, 2.0, 3.0]
@@ -1,73 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import weakref
import pytest
from vllm import LLM, PoolingRequestOutput
from vllm.config import PoolerConfig
from vllm.tasks import PoolingTask
MODEL_NAME = "jason9693/Qwen2.5-1.5B-apeach"
prompt = "The chef prepared a delicious meal."
prompt_token_ids = [785, 29706, 10030, 264, 17923, 15145, 13]
num_labels = 2
@pytest.fixture(scope="module")
def llm(vllm_runner):
with vllm_runner(
MODEL_NAME,
max_model_len=None,
pooler_config=PoolerConfig(task="token_classify"),
max_num_batched_tokens=32768,
tensor_parallel_size=1,
gpu_memory_utilization=0.75,
enforce_eager=True,
seed=0,
enable_chunked_prefill=None,
) as runner:
# pytest caches yielded fixtures until after teardown, so use a proxy to
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
yield weakref.proxy(runner.llm)
@pytest.mark.skip_global_cleanup
def test_str_prompts(llm: LLM):
outputs = llm.encode(prompt, pooling_task="token_classify", use_tqdm=False)
assert len(outputs) == 1
assert isinstance(outputs[0], PoolingRequestOutput)
assert outputs[0].prompt_token_ids == prompt_token_ids
assert outputs[0].outputs.data.shape == (len(prompt_token_ids), num_labels)
@pytest.mark.skip_global_cleanup
def test_token_ids_prompts(llm: LLM):
outputs = llm.encode(
[prompt_token_ids], pooling_task="token_classify", use_tqdm=False
)
assert len(outputs) == 1
assert isinstance(outputs[0], PoolingRequestOutput)
assert outputs[0].prompt_token_ids == prompt_token_ids
assert outputs[0].outputs.data.shape == (len(prompt_token_ids), num_labels)
@pytest.mark.skip_global_cleanup
def test_score_api(llm: LLM):
err_msg = "This model does not support the Scoring API."
with pytest.raises(ValueError, match=err_msg):
llm.score("ping", "pong", use_tqdm=False)
@pytest.mark.parametrize("task", ["classify", "embed", "token_embed", "plugin"])
def test_unsupported_tasks(llm: LLM, task: PoolingTask, caplog_vllm):
if task == "plugin":
err_msg = "No IOProcessor plugin installed."
elif task == "classify":
err_msg = "Try switching the model's pooling_task via.+"
else:
err_msg = "Embedding API is not supported by this model.+"
with pytest.raises(ValueError, match=err_msg):
llm.encode(prompt, pooling_task=task, use_tqdm=False)
@@ -1,75 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import requests
from tests.utils import RemoteOpenAIServer
from vllm.entrypoints.pooling.pooling.protocol import PoolingResponse
MODEL_NAME = "jason9693/Qwen2.5-1.5B-apeach"
DTYPE = "float32" # Use float32 to avoid NaN issue
input_text = "This product was excellent and exceeded my expectations"
input_tokens = [1986, 1985, 572, 9073, 323, 33808, 847, 16665]
@pytest.fixture(scope="module")
def server():
args = [
"--enforce-eager",
"--max-model-len",
"512",
"--dtype",
DTYPE,
"--pooler-config.task",
"token_classify",
]
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
yield remote_server
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
async def test_pooling_token_classify(server: RemoteOpenAIServer, model_name: str):
task = "token_classify"
response = requests.post(
server.url_for("pooling"),
json={
"model": model_name,
"input": input_text,
"encoding_format": "float",
"task": task,
},
)
poolings = PoolingResponse.model_validate(response.json())
assert len(poolings.data) == 1
assert len(poolings.data[0].data) == 8
assert isinstance(poolings.data[0].data[0], list)
assert len(poolings.data[0].data[0]) == 2
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
@pytest.mark.parametrize("task", ["classify", "embed", "token_embed", "plugin"])
async def test_pooling_not_supported(
server: RemoteOpenAIServer, model_name: str, task: str
):
response = requests.post(
server.url_for("pooling"),
json={
"model": model_name,
"input": input_text,
"encoding_format": "float",
"task": task,
},
)
assert response.json()["error"]["type"] == "BadRequestError"
if task == "plugin":
err_msg = "No IOProcessor plugin installed."
elif task == "classify":
err_msg = "Try switching the model's pooling_task via"
else:
err_msg = f"Unsupported task: {task!r}"
assert response.json()["error"]["message"].startswith(err_msg)
@@ -1,282 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import weakref
import pytest
import torch
from tests.models.utils import check_embeddings_close
from vllm import LLM, PoolingParams, PoolingRequestOutput
from vllm.config import PoolerConfig
from vllm.exceptions import VLLMValidationError
from vllm.pooling_params import LateChunkingParams
from vllm.tasks import PoolingTask
MODEL_NAME = "intfloat/multilingual-e5-small"
prompt = "The chef prepared a delicious meal."
prompt_token_ids = [0, 581, 21861, 133888, 10, 8, 150, 60744, 109911, 5, 2]
embedding_size = 384
@pytest.fixture(scope="module")
def llm(vllm_runner):
with vllm_runner(
MODEL_NAME,
max_model_len=None,
pooler_config=PoolerConfig(task="token_embed"),
max_num_batched_tokens=32768,
tensor_parallel_size=1,
gpu_memory_utilization=0.75,
enforce_eager=True,
seed=0,
enable_chunked_prefill=None,
) as runner:
assert embedding_size == runner.llm.model_config.embedding_size
# pytest caches yielded fixtures until after teardown, so use a proxy to
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
yield weakref.proxy(runner.llm)
@pytest.mark.skip_global_cleanup
def test_str_prompts(llm: LLM):
outputs = llm.encode(prompt, pooling_task="token_embed", use_tqdm=False)
assert len(outputs) == 1
assert isinstance(outputs[0], PoolingRequestOutput)
assert outputs[0].outputs.data.shape == (11, 384)
@pytest.mark.skip_global_cleanup
def test_token_ids_prompts(llm: LLM):
outputs = llm.encode([prompt_token_ids], pooling_task="token_embed", use_tqdm=False)
assert len(outputs) == 1
assert isinstance(outputs[0], PoolingRequestOutput)
assert outputs[0].outputs.data.shape == (11, 384)
@pytest.mark.parametrize("task", ["embed", "classify", "token_classify", "plugin"])
def test_unsupported_tasks(llm: LLM, task: PoolingTask, caplog_vllm):
if task == "plugin":
err_msg = "No IOProcessor plugin installed."
elif task == "embed":
err_msg = "Try switching the model's pooling_task via.+"
else:
err_msg = "Classification API is not supported by this model.+"
with pytest.raises(ValueError, match=err_msg):
llm.encode(prompt, pooling_task=task, use_tqdm=False)
def test_bge_m3_late_chunking_keeps_special_tokens(vllm_runner):
with vllm_runner(
"BAAI/bge-m3",
hf_overrides={"architectures": ["BgeM3EmbeddingModel"]},
pooler_config=PoolerConfig(task="token_embed"),
dtype="float32",
max_model_len=128,
# Keep encoder batch size fixed for the numerical reference.
max_num_seqs=1,
gpu_memory_utilization=0.5,
enforce_eager=True,
) as runner:
llm = runner.llm
text = "A document about Berlin and its museums."
sizes = (1, 3, 128, None)
tokens, *outputs = llm.encode(
[text] * len(sizes),
pooling_task="token_embed",
pooling_params=[
PoolingParams(
use_activation=size not in (1, None),
late_chunking_params=LateChunkingParams(size) if size else None,
)
for size in sizes
],
use_tqdm=False,
)
assert tokens.outputs.data.shape[0] == len(tokens.prompt_token_ids)
assert tokens.late_chunking.chunks[0].char_range is None
for size, output in zip(sizes[1:], outputs):
if size is None:
# Ordinary BGE-M3 token embeddings still drop BOS only.
expected = tokens.outputs.data[1:]
assert output.late_chunking is None
else:
# BGE-M3's projector is linear, so its unnormalized token
# outputs also provide a reference for projected chunk means.
expected = torch.nn.functional.normalize(
torch.stack([p.mean(0) for p in tokens.outputs.data.split(size)]),
dim=-1,
)
assert len(output.late_chunking.chunks) == len(expected)
torch.testing.assert_close(
output.outputs.data, expected, atol=2e-5, rtol=2e-4
)
def test_nomic_late_chunking_without_special_tokens(vllm_runner):
"""Keep source ranges without special tokens and reject empty tokenization."""
with vllm_runner(
"nomic-ai/nomic-embed-text-v1",
revision="720244025c1a7e15661a174c63cce63c8218e52b",
trust_remote_code=True,
runner="pooling",
pooler_config=PoolerConfig(task="token_embed"),
dtype="float32",
max_model_len=128,
max_num_seqs=2,
gpu_memory_utilization=0.3,
enforce_eager=True,
enable_prefix_caching=False,
enable_chunked_prefill=False,
) as runner:
llm = runner.llm
text = "search_document: 中文 😀 café e\u0301"
params = PoolingParams(
late_chunking_params=LateChunkingParams(3), use_activation=False
)
kwargs = {"add_special_tokens": False}
raw = llm.encode(
text,
pooling_task="token_embed",
use_tqdm=False,
pooling_params=PoolingParams(use_activation=False),
tokenization_kwargs=kwargs,
)[0]
actual = llm.encode(
text,
pooling_task="token_embed",
use_tqdm=False,
pooling_params=params,
tokenization_kwargs=kwargs,
)[0]
expected = torch.stack([part.mean(0) for part in raw.outputs.data.split(3)])
torch.testing.assert_close(actual.outputs.data, expected, atol=1e-5, rtol=1e-4)
encoded = llm.get_tokenizer()(text, return_offsets_mapping=True, **kwargs)
assert actual.prompt_token_ids == encoded["input_ids"]
for chunk in actual.late_chunking.chunks:
start, end = chunk.token_range
offsets = [(a, b) for a, b in encoded["offset_mapping"][start:end] if a < b]
assert chunk.char_range == (offsets[0][0], offsets[-1][1])
for rejected in ("long " * 256, " "):
with pytest.raises(VLLMValidationError):
llm.encode(
rejected,
pooling_task="token_embed",
use_tqdm=False,
pooling_params=params,
tokenization_kwargs=kwargs,
)
assert (
llm.encode(text, pooling_task="token_embed", use_tqdm=False)[
0
].late_chunking
is None
)
@pytest.mark.skip_global_cleanup
@pytest.mark.parametrize("normalize", [False, True])
def test_e5_late_chunking_mixed_requests(llm: LLM, normalize: bool):
texts = [prompt, "Short text.", "中文 😀 café"]
raw = llm.encode(
texts,
pooling_task="token_embed",
use_tqdm=False,
pooling_params=PoolingParams(use_activation=False),
)
sizes = [3, None, 100]
params = [
PoolingParams(
late_chunking_params=LateChunkingParams(size) if size is not None else None,
use_activation=normalize,
)
for size in sizes
]
actual = llm.encode(
texts, pooling_task="token_embed", pooling_params=params, use_tqdm=False
)
for original, result, size in zip(raw, actual, sizes):
expected = original.outputs.data
if size is not None:
expected = torch.stack(
[part.float().mean(0) for part in expected.split(size)]
)
assert result.late_chunking is not None
assert len(result.late_chunking.chunks) == len(expected)
assert result.late_chunking.input_tokens == len(result.prompt_token_ids)
else:
assert result.late_chunking is None
if normalize:
expected = torch.nn.functional.normalize(expected, dim=-1)
check_embeddings_close(
embeddings_0_lst=result.outputs.data.tolist(),
embeddings_1_lst=expected.tolist(),
name_0="late_chunking",
name_1="token_reference",
)
torch.testing.assert_close(
result.outputs.data.norm(dim=-1),
expected.norm(dim=-1),
atol=1e-3,
rtol=1e-3,
)
@pytest.mark.parametrize("use_v2_runner", [False, True])
def test_late_chunking_recomputes_cached_prefix_with_chunked_prefill(
vllm_runner, monkeypatch, use_v2_runner
):
# Dummy weights suffice for testing scheduling and reduction of the same states.
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1" if use_v2_runner else "0")
with vllm_runner(
"openai-community/gpt2",
runner="pooling",
convert="embed",
load_format="dummy",
pooler_config=PoolerConfig(
task="token_embed", seq_pooling_type="LAST", tok_pooling_type="ALL"
),
dtype="float32",
max_model_len=128,
max_num_batched_tokens=16,
max_num_seqs=2,
enable_prefix_caching=True,
enable_chunked_prefill=True,
enforce_eager=True,
gpu_memory_utilization=0.3,
seed=0,
) as runner:
text = "The document has several important words. " * 6
llm = runner.llm
raw = llm.encode(
text,
pooling_task="token_embed",
use_tqdm=False,
pooling_params=PoolingParams(use_activation=False),
)[0]
assert len(raw.prompt_token_ids) > 16
cached = llm.encode(
text,
pooling_task="token_embed",
use_tqdm=False,
pooling_params=PoolingParams(
use_activation=False, skip_reading_prefix_cache=False
),
)[0]
assert cached.num_cached_tokens > 0
params = PoolingParams(
late_chunking_params=LateChunkingParams(7),
use_activation=False,
skip_reading_prefix_cache=False,
)
actual = llm.encode(
text, pooling_task="token_embed", pooling_params=params, use_tqdm=False
)[0]
assert actual.num_cached_tokens == 0
assert actual.prompt_token_ids == raw.prompt_token_ids
expected = torch.stack([part.mean(0) for part in raw.outputs.data.split(7)])
torch.testing.assert_close(actual.outputs.data, expected, atol=1e-5, rtol=1e-4)
assert actual.late_chunking.input_tokens == len(raw.prompt_token_ids)
assert params.skip_reading_prefix_cache is False
assert params.late_chunking_params is not None
@@ -1,101 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import requests
from tests.utils import RemoteOpenAIServer
from vllm.entrypoints.pooling.pooling.protocol import PoolingResponse
MODEL_NAME = "intfloat/multilingual-e5-small"
DTYPE = "bfloat16"
input_text = "The best thing about vLLM is that it supports many different models"
input_tokens = [
0,
581,
2965,
13580,
1672,
81,
23708,
594,
83,
450,
442,
8060,
7,
5941,
12921,
115774,
2,
]
@pytest.fixture(scope="module")
def server():
args = [
"--runner",
"pooling",
"--dtype",
DTYPE,
"--enforce-eager",
"--max-model-len",
"512",
"--pooler-config.task",
"token_embed",
]
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
yield remote_server
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
async def test_pooling_token_embed(server: RemoteOpenAIServer, model_name: str):
task = "token_embed"
request_id = "client-request-id"
response = requests.post(
server.url_for("pooling"),
json={
"model": model_name,
"input": input_text,
"encoding_format": "float",
"task": task,
"request_id": request_id,
},
)
poolings = PoolingResponse.model_validate(response.json())
assert poolings.id == f"pooling-{request_id}"
assert len(poolings.data) == 1
assert len(poolings.data[0].data) == len(input_tokens)
assert isinstance(poolings.data[0].data[0], list)
assert len(poolings.data[0].data[0]) == 384
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
@pytest.mark.parametrize("task", ["embed", "classify", "token_classify", "plugin"])
async def test_pooling_not_supported(
server: RemoteOpenAIServer, model_name: str, task: str
):
response = requests.post(
server.url_for("pooling"),
json={
"model": model_name,
"input": "test",
"encoding_format": "float",
"task": task,
},
)
assert response.json()["error"]["type"] == "BadRequestError"
if task == "plugin":
err_msg = "No IOProcessor plugin installed."
elif task == "embed":
err_msg = "Try switching the model's pooling_task via"
else:
err_msg = f"Unsupported task: {task!r}"
assert response.json()["error"]["message"].startswith(err_msg)
@@ -1,218 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from http import HTTPStatus
from unittest.mock import MagicMock
import pytest
from vllm import PoolingParams
from vllm.config import ModelConfig
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.openai.models.protocol import BaseModelPath
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.pooling.base.serving import PoolingBaseServing
from vllm.entrypoints.pooling.typing import PoolingServeContext
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.lora.protocol import (
LoadLoRAAdapterRequest,
UnloadLoRAAdapterRequest,
)
from vllm.exceptions import VLLMNotFoundError
from vllm.lora.request import LoRARequest
MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
BASE_MODEL_PATHS = [BaseModelPath(name=MODEL_NAME, model_path=MODEL_NAME)]
LORA_LOADING_SUCCESS_MESSAGE = "Success: LoRA adapter '{lora_name}' added successfully."
LORA_UNLOADING_SUCCESS_MESSAGE = (
"Success: LoRA adapter '{lora_name}' removed successfully."
)
async def _async_serving_models_init_with_mock() -> tuple[
OpenAIServingModels, MagicMock
]:
mock_engine_client = MagicMock(spec=EngineClient)
# Set the max_model_len attribute to avoid missing attribute
mock_model_config = MagicMock(spec=ModelConfig)
mock_model_config.max_model_len = 2048
mock_engine_client.model_config = mock_model_config
mock_engine_client.input_processor = MagicMock()
mock_engine_client.renderer = MagicMock()
serving_models = OpenAIServingModels(
engine_client=mock_engine_client,
base_model_paths=BASE_MODEL_PATHS,
lora_modules=None,
)
await serving_models.init_static_loras()
# The mock is returned separately: `engine_client` is typed as the protocol,
# so assertion helpers are not visible through it.
return serving_models, mock_engine_client
async def _async_serving_models_init() -> OpenAIServingModels:
serving_models, _ = await _async_serving_models_init_with_mock()
return serving_models
@pytest.mark.asyncio
async def test_serving_model_name():
serving_models = await _async_serving_models_init()
assert serving_models.model_name(None) == MODEL_NAME
request = LoRARequest(
lora_name="adapter", lora_path="/path/to/adapter2", lora_int_id=1
)
assert serving_models.model_name(request) == request.lora_name
@pytest.mark.asyncio
async def test_load_lora_adapter_success():
serving_models = await _async_serving_models_init()
request = LoadLoRAAdapterRequest(lora_name="adapter", lora_path="/path/to/adapter2")
response = await serving_models.load_lora_adapter(request)
assert response == LORA_LOADING_SUCCESS_MESSAGE.format(lora_name="adapter")
assert len(serving_models.lora_requests) == 1
assert "adapter" in serving_models.lora_requests
assert serving_models.lora_requests["adapter"].lora_name == "adapter"
@pytest.mark.asyncio
async def test_load_lora_adapter_missing_fields():
serving_models = await _async_serving_models_init()
request = LoadLoRAAdapterRequest(lora_name="", lora_path="")
response = await serving_models.load_lora_adapter(request)
assert isinstance(response, ErrorResponse)
assert response.error.type == "InvalidUserInput"
assert response.error.code == HTTPStatus.BAD_REQUEST
@pytest.mark.asyncio
async def test_load_lora_adapter_duplicate():
serving_models = await _async_serving_models_init()
request = LoadLoRAAdapterRequest(
lora_name="adapter1", lora_path="/path/to/adapter1"
)
response = await serving_models.load_lora_adapter(request)
assert response == LORA_LOADING_SUCCESS_MESSAGE.format(lora_name="adapter1")
assert len(serving_models.lora_requests) == 1
request = LoadLoRAAdapterRequest(
lora_name="adapter1", lora_path="/path/to/adapter1"
)
response = await serving_models.load_lora_adapter(request)
assert isinstance(response, ErrorResponse)
assert response.error.type == "InvalidUserInput"
assert response.error.code == HTTPStatus.BAD_REQUEST
assert len(serving_models.lora_requests) == 1
@pytest.mark.asyncio
@pytest.mark.parametrize("load_inplace", [False, True])
async def test_load_lora_adapter_base_model_name(load_inplace: bool):
"""A LoRA named after a served model would shadow it for all requests."""
serving_models, mock_engine_client = await _async_serving_models_init_with_mock()
request = LoadLoRAAdapterRequest(
lora_name=MODEL_NAME, lora_path="/path/to/adapter", load_inplace=load_inplace
)
response = await serving_models.load_lora_adapter(request)
assert isinstance(response, ErrorResponse)
assert response.error.type == "InvalidUserInput"
assert response.error.code == HTTPStatus.BAD_REQUEST
assert len(serving_models.lora_requests) == 0
mock_engine_client.add_lora.assert_not_called()
@pytest.mark.asyncio
async def test_unload_lora_adapter_success():
serving_models = await _async_serving_models_init()
request = LoadLoRAAdapterRequest(
lora_name="adapter1", lora_path="/path/to/adapter1"
)
response = await serving_models.load_lora_adapter(request)
assert len(serving_models.lora_requests) == 1
unload_request = UnloadLoRAAdapterRequest(lora_name="adapter1")
response = await serving_models.unload_lora_adapter(unload_request)
assert response == LORA_UNLOADING_SUCCESS_MESSAGE.format(lora_name="adapter1")
assert len(serving_models.lora_requests) == 0
@pytest.mark.asyncio
async def test_unload_lora_adapter_missing_fields():
serving_models = await _async_serving_models_init()
request = UnloadLoRAAdapterRequest(lora_name="", lora_int_id=None)
response = await serving_models.unload_lora_adapter(request)
assert isinstance(response, ErrorResponse)
assert response.error.type == "InvalidUserInput"
assert response.error.code == HTTPStatus.BAD_REQUEST
@pytest.mark.asyncio
async def test_unload_lora_adapter_not_found():
serving_models = await _async_serving_models_init()
request = UnloadLoRAAdapterRequest(lora_name="nonexistent_adapter")
response = await serving_models.unload_lora_adapter(request)
assert isinstance(response, ErrorResponse)
assert response.error.type == "NotFoundError"
assert response.error.code == HTTPStatus.NOT_FOUND
class _ConcretePoolingServing(PoolingBaseServing):
"""Minimal concrete subclass used only in these unit tests."""
request_id_prefix = "test"
def get_io_processor(self, request):
raise NotImplementedError
def _build_response(self, ctx):
raise NotImplementedError
def _make_pooling_serving(lora_name: str) -> _ConcretePoolingServing:
lora_request = LoRARequest(
lora_name=lora_name, lora_int_id=1, lora_path="/path/to/lora"
)
mock_models = MagicMock()
mock_models.lora_requests = {lora_name: lora_request}
mock_models.is_base_model.side_effect = lambda name: name == MODEL_NAME
serving = object.__new__(_ConcretePoolingServing)
serving.models = mock_models
return serving
def _make_pooling_ctx(
model_name: str, serving: PoolingBaseServing
) -> PoolingServeContext:
mock_request = MagicMock()
mock_request.model = model_name
return PoolingServeContext(
request=mock_request,
model_name=MODEL_NAME,
request_id="test-id",
pooling_params=PoolingParams(),
lora_request=serving._maybe_get_adapters(mock_request),
priorities=0,
prompt_extras=None,
)
def test_pooling_maybe_get_adapters_lora_name_sets_lora_request():
"""LoRA adapter name must populate ctx.lora_request without raising."""
lora_name = "bot-embed-lora"
serving = _make_pooling_serving(lora_name)
ctx = _make_pooling_ctx(lora_name, serving)
assert ctx.lora_request is not None
assert ctx.lora_request.lora_name == lora_name
def test_pooling_maybe_get_adapters_unknown_model_raises():
"""An unrecognised model name must still raise VLLMNotFoundError."""
serving = _make_pooling_serving("some-lora")
with pytest.raises(VLLMNotFoundError):
_make_pooling_ctx("unknown-model", serving)
@@ -1,402 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Evaluate Transcription API correctness by computing Word Error Rate (WER)
on a given ASR dataset. When provided, it will also compare the WER against
a baseline.
This simulates real work usage of the API and makes sure that the frontend and
AsyncLLMEngine are working correctly.
"""
import asyncio
import io
import time
from statistics import mean, median
import pytest
import soundfile
import torch
from datasets import Audio, load_dataset
from jiwer import wer
from transformers.models.whisper.english_normalizer import EnglishTextNormalizer
from vllm.benchmarks.datasets.datasets import ASRDataset
from vllm.multimodal.audio import get_audio_duration
from vllm.tokenizers import get_tokenizer
from ....models.registry import HF_EXAMPLE_MODELS
from ....utils import RemoteOpenAIServer
# Tuned to prevent OOM on 18GB GPUs in transcription correctness tests.
MAX_SEQS_FOR_TRANSCRIPTION_TEST = 8
GPU_UTIL_FOR_TRANSCRIPTION_TEST = 0.5
def to_bytes(y, sr):
buffer = io.BytesIO()
soundfile.write(buffer, y, sr, format="WAV")
buffer.seek(0)
return buffer
def load_audio_sample(audio):
# Avoid torchcodec in CI by decoding dataset audio with soundfile.
if "array" in audio and "sampling_rate" in audio:
return audio["array"], audio["sampling_rate"]
if audio.get("path"):
return soundfile.read(audio["path"], dtype="float32")
if audio.get("bytes") is not None:
return soundfile.read(io.BytesIO(audio["bytes"]), dtype="float32")
raise ValueError("Audio sample did not contain array, path, or bytes data")
# not all models have a normalizer so use the one from whisper as a standard option
normalizer_model_info = HF_EXAMPLE_MODELS.find_hf_info("openai/whisper-large-v3")
normalizer_tokenizer = get_tokenizer(
"openai/whisper-large-v3",
tokenizer_mode=normalizer_model_info.tokenizer_mode,
trust_remote_code=normalizer_model_info.trust_remote_code,
)
normalizer = EnglishTextNormalizer(normalizer_tokenizer.english_spelling_normalizer)
async def transcribe_audio(client, tokenizer, y, sr, extra_body=None):
# Send loaded audio directly instead of loading from disk,
# don't account for that time though
with to_bytes(y, sr) as f:
start_time = time.perf_counter()
transcription = await client.audio.transcriptions.create(
file=f,
model=tokenizer.name_or_path,
language="en",
temperature=0.0,
extra_body=extra_body,
)
end_time = time.perf_counter()
# NOTE there's no streaming in transcriptions, can't measure ttft
latency = end_time - start_time
num_output_tokens = len(
tokenizer(transcription.text, add_special_tokens=False).input_ids
)
return latency, num_output_tokens, transcription.text
async def bound_transcribe(
sem, client, tokenizer, audio, sr, reference, extra_body=None
):
# Use semaphore to limit concurrent requests.
async with sem:
result = await transcribe_audio(
client, tokenizer, audio, sr, extra_body=extra_body
)
# Normalize *english* output/reference for evaluation.
out = normalizer(result[2])
ref = normalizer(reference)
return result[:2] + (out, ref)
async def process_dataset(model, client, data, concurrent_request, extra_body=None):
sem = asyncio.Semaphore(concurrent_request)
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
tokenizer = get_tokenizer(
model,
tokenizer_mode=model_info.tokenizer_mode,
trust_remote_code=model_info.trust_remote_code,
)
# Warmup call as the first `load_audio` server-side is quite slow.
audio, sr = load_audio_sample(data[0]["audio"])
_ = await bound_transcribe(sem, client, tokenizer, audio, sr, "", extra_body)
tasks: list[asyncio.Task] = []
for sample in data:
audio, sr = load_audio_sample(sample["audio"])
task = asyncio.create_task(
bound_transcribe(
sem, client, tokenizer, audio, sr, sample["text"], extra_body
)
)
tasks.append(task)
return await asyncio.gather(*tasks)
def print_performance_metrics(results, total_time):
latencies = [res[0] for res in results]
total_tokens = sum([res[1] for res in results])
total = len(results)
print(f"Total Requests: {total}")
print(f"Successful Requests: {len(latencies)}")
print(f"Average Latency: {mean(latencies):.4f} seconds")
print(f"Median Latency: {median(latencies):.4f} seconds")
perc = sorted(latencies)[int(len(latencies) * 0.95) - 1]
print(f"95th Percentile Latency: {perc:.4f} seconds")
# Throughput
req_throughput = len(latencies) / total_time
print(f"Estimated req_Throughput: {req_throughput:.2f} requests/s")
throughput = total_tokens / total_time
print(f"Estimated Throughput: {throughput:.2f} tok/s")
def add_duration(sample):
y, sr = load_audio_sample(sample["audio"])
sample["duration_ms"] = get_audio_duration(y=y, sr=sr) * 1000
return sample
def load_asr_dataset_rows(dataset_repo: str, split="validation", **hf_kwargs):
if dataset_repo in ASRDataset.SUPPORTED_DATASET_PATHS:
asr_dataset_kwargs = {
"dataset_path": dataset_repo,
"dataset_split": split,
"disable_shuffle": True,
"no_stream": True,
}
for key in ("dataset_subset", "hf_name", "trust_remote_code"):
if key in hf_kwargs:
asr_dataset_kwargs[key] = hf_kwargs[key]
return ASRDataset(**asr_dataset_kwargs).data
return load_dataset(dataset_repo, split=split, **hf_kwargs)
def load_shortform_eval_dataset(dataset_repo: str, split="validation", **hf_kwargs):
## Load and filter the dataset.
dataset = load_asr_dataset_rows(dataset_repo, split=split, **hf_kwargs)
dataset = dataset.cast_column("audio", Audio(decode=False))
if "duration_ms" not in dataset.column_names:
# Compute duration to filter.
dataset = dataset.map(add_duration)
# Whisper max supported duration.
dataset = dataset.filter(lambda example: example["duration_ms"] < 30000)
return dataset
def run_evaluation(
model: str,
client,
dataset,
max_concurrent_reqs: int,
n_examples: int = -1,
print_metrics: bool = True,
extra_body=None,
):
if n_examples > 0:
dataset = dataset.select(range(n_examples))
start = time.perf_counter()
results = asyncio.run(
process_dataset(
model, client, dataset, max_concurrent_reqs, extra_body=extra_body
)
)
end = time.perf_counter()
total_time = end - start
print(f"Total Test Time: {total_time:.4f} seconds")
if print_metrics:
print_performance_metrics(results, total_time)
# Compute WER
predictions = [res[2] for res in results]
references = [res[3] for res in results]
wer_score = 100 * wer(references, predictions)
print("WER:", wer_score)
return wer_score
LONGFORM_DATASET_REPO = ASRDataset.EARNINGS22_CLEANED_DATASET
LONGFORM_DATASET_SPLIT = "test"
LONGFORM_NUM_SAMPLES = 6
def load_longform_dataset():
dataset = load_asr_dataset_rows(
LONGFORM_DATASET_REPO,
split=LONGFORM_DATASET_SPLIT,
)
assert len(dataset) >= LONGFORM_NUM_SAMPLES
return dataset.select(range(LONGFORM_NUM_SAMPLES))
async def transcribe_audio_path(client, tokenizer, audio_path: str, extra_body=None):
with open(audio_path, "rb") as f:
start_time = time.perf_counter()
transcription = await client.audio.transcriptions.create(
file=f,
model=tokenizer.name_or_path,
language="en",
temperature=0.0,
extra_body=extra_body,
)
end_time = time.perf_counter()
latency = end_time - start_time
num_output_tokens = len(
tokenizer(transcription.text, add_special_tokens=False).input_ids
)
return latency, num_output_tokens, transcription.text
async def bound_transcribe_path(
sem, client, tokenizer, audio_path, reference, extra_body=None
):
async with sem:
result = await transcribe_audio_path(
client, tokenizer, audio_path, extra_body=extra_body
)
out = normalizer(result[2])
ref = normalizer(reference)
return result[:2] + (out, ref)
async def process_longform_dataset(
model, client, data, concurrent_request, extra_body=None
):
sem = asyncio.Semaphore(concurrent_request)
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
tokenizer = get_tokenizer(
model,
tokenizer_mode=model_info.tokenizer_mode,
trust_remote_code=model_info.trust_remote_code,
)
warmup_path = data[0]["audio"]["path"]
_ = await bound_transcribe_path(sem, client, tokenizer, warmup_path, "", extra_body)
tasks: list[asyncio.Task] = []
for sample in data:
audio_path = sample["audio"]["path"]
task = asyncio.create_task(
bound_transcribe_path(
sem, client, tokenizer, audio_path, sample["text"], extra_body
)
)
tasks.append(task)
return await asyncio.gather(*tasks)
def run_longform_evaluation(
model: str,
client,
dataset,
max_concurrent_reqs: int,
print_metrics: bool = True,
extra_body=None,
):
start = time.perf_counter()
results = asyncio.run(
process_longform_dataset(
model, client, dataset, max_concurrent_reqs, extra_body=extra_body
)
)
end = time.perf_counter()
total_time = end - start
print(f"Total Test Time: {total_time:.4f} seconds")
if print_metrics:
print_performance_metrics(results, total_time)
predictions = [res[2] for res in results]
references = [res[3] for res in results]
wer_score = 100 * wer(references, predictions)
print("WER:", wer_score)
return wer_score
# alternatives "openai/whisper-large-v2", "openai/whisper-large-v3-turbo"..
# NOTE: Expected WER measured with equivalent hf.transformers args:
# whisper-large-v3 + esb-datasets-earnings22-validation-tiny-filtered.
@pytest.mark.parametrize(
"model_config",
[
("openai/whisper-large-v3", 12.744980),
# CohereASR is used to test the variable encoder length code paths
("CohereLabs/cohere-transcribe-03-2026", 11.92),
],
)
# Original dataset is 20GB+ in size, hence we use a pre-filtered slice.
@pytest.mark.parametrize(
"dataset_repo", ["D4nt3/esb-datasets-earnings22-validation-tiny-filtered"]
)
def test_wer_correctness(
model_config, dataset_repo, n_examples=-1, max_concurrent_request=None
):
model_name, expected_wer = model_config
model_info = HF_EXAMPLE_MODELS.find_hf_info(model_name)
server_args = [
"--enforce-eager",
f"--tokenizer_mode={model_info.tokenizer_mode}",
f"--max_num_seqs={MAX_SEQS_FOR_TRANSCRIPTION_TEST}",
f"--gpu_memory_utilization={GPU_UTIL_FOR_TRANSCRIPTION_TEST}",
]
if model_info.trust_remote_code:
server_args.append("--trust-remote-code")
with RemoteOpenAIServer(
model_name,
server_args,
) as remote_server:
dataset = load_shortform_eval_dataset(dataset_repo)
if not max_concurrent_request:
# No max concurrency
max_concurrent_request = n_examples if n_examples > 0 else len(dataset)
client = remote_server.get_async_client()
wer = run_evaluation(
model_name,
client,
dataset,
max_concurrent_request,
n_examples,
)
print(f"Expected WER: {expected_wer}, Actual WER: {wer}")
if expected_wer:
wer_atol, wer_rtol = 1e-1, 1e-2
max_wer = expected_wer + wer_atol + wer_rtol * abs(expected_wer)
assert wer <= max_wer, (
f"WER {wer:.6f} exceeds maximum allowed {max_wer:.6f} "
f"(baseline {expected_wer:.6f})"
)
# 14-22mins of 6 audio samples of total ~115 mins and just 37MB.
# checks for long audio transcription correctness and RMS split.
@pytest.mark.parametrize(
"model_config",
[("openai/whisper-large-v3", 9.5)],
)
def test_long_audio_wer_correctness(model_config):
model_name, expected_wer = model_config
model_info = HF_EXAMPLE_MODELS.find_hf_info(model_name)
server_args = [
f"--tokenizer_mode={model_info.tokenizer_mode}",
]
if model_info.trust_remote_code:
server_args.append("--trust-remote-code")
# 1800 seconds is 30 minutes
env_dict = {
"VLLM_MAX_AUDIO_DECODE_DURATION_S": "1800",
}
with RemoteOpenAIServer(
model_name,
server_args,
env_dict=env_dict,
) as remote_server:
dataset = load_longform_dataset()
client = remote_server.get_async_client()
wer = run_longform_evaluation(
model=model_name,
client=client,
dataset=dataset,
max_concurrent_reqs=LONGFORM_NUM_SAMPLES,
)
print(f"Expected WER: {expected_wer}, Actual WER: {wer}")
torch.testing.assert_close(wer, expected_wer, atol=1e-1, rtol=1e-2)
@@ -1,64 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for ``Qwen3ASR``'s user-text sanitizer.
The sanitizer is the security boundary between user-supplied transcription
fields (``prompt`` / ``response_prefix``) and the structured ChatML prompt
template. It must strip both ``<|...|>`` control tokens and the
``<asr_text>`` assistant-prefix delimiter, and it must do so to a fixpoint
so nested payloads cannot reconstruct a valid token after a single pass.
"""
import pytest
from vllm.model_executor.models.qwen3_asr import _sanitize_transcription_user_text
@pytest.mark.parametrize(
("text", "expected"),
[
# No-op cases
("", ""),
("plain text", "plain text"),
("|piped|content", "|piped|content"),
("contains < and > but not as a token", "contains < and > but not as a token"),
# Single-pass strips
("<|im_end|>", ""),
("<|im_start|>assistant<|im_end|>", "assistant"),
("a<|x|>b", "ab"),
("foo<asr_text>bar", "foobar"),
# Nested ChatML reconstruction attacks (would bypass a single re.sub)
("<|im<|x|>_end|>", ""),
("<|<|inner|>middle<|x|>_end|>", ""),
# Nested <asr_text> reconstruction attack
# (would bypass a single str.replace)
("<asr_te<asr_text>xt>", ""),
("<asr_te<asr_te<asr_text>xt>xt>", ""),
# Combined attacks across both kinds of token
("<|im_end|>foo<asr_text>bar<|<|x|>im_end|>", "foobar"),
("foo<asr_te<|x|>xt>bar", "foobar"),
],
)
def test_sanitize_strips_control_tokens(text: str, expected: str) -> None:
assert _sanitize_transcription_user_text(text) == expected
def test_sanitize_handles_falsy_inputs() -> None:
assert _sanitize_transcription_user_text("") == ""
# The dataclass default for ``response_prefix`` is the empty string;
# the sanitizer must accept that without exception or extra work.
assert _sanitize_transcription_user_text(None) == ""
def test_sanitize_is_idempotent() -> None:
"""Once sanitized, applying again must be a no-op (fixpoint property)."""
cases = [
"plain text",
"<|im<|x|>_end|>",
"<asr_te<asr_text>xt>",
"<|im_end|>foo<asr_text>bar<|<|x|>im_end|>",
]
for raw in cases:
once = _sanitize_transcription_user_text(raw)
twice = _sanitize_transcription_user_text(once)
assert once == twice, f"not idempotent for {raw!r}"
@@ -1,390 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""ASR inter-chunk spacing: ``asr_inter_chunk_separator`` and transcription
serving (mocked).
Unit tests cover the helper and ``SupportsTranscription.no_space_languages``.
Integration-style tests exercise ``OpenAIServingTranscription`` streaming and
``create_transcription`` without loading a model.
"""
from __future__ import annotations
import json
from collections.abc import AsyncGenerator
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from vllm.config import ModelConfig
from vllm.config.speech_to_text import SpeechToTextConfig
from vllm.entrypoints.generate.base.protocol import RequestResponseMetadata
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.entrypoints.speech_to_text.base.serving import (
SpeechToTextBaseServing,
asr_inter_chunk_separator,
)
from vllm.entrypoints.speech_to_text.transcription.protocol import TranscriptionRequest
from vllm.entrypoints.speech_to_text.transcription.serving import (
OpenAIServingTranscription,
)
from vllm.model_executor.models.interfaces import (
StreamingTranscriptionPostProcessor,
SupportsTranscription,
)
from vllm.model_executor.models.qwen3_asr import Qwen3ASRForConditionalGeneration
from vllm.outputs import CompletionOutput, RequestOutput
# --- Unit: helper + protocol -------------------------------------------------
def test_default_no_space_languages_includes_zh_and_ja():
assert SupportsTranscription.no_space_languages == {"ja", "zh"}
@pytest.mark.parametrize(
("language", "expected_sep"),
[
("en", " "),
("EN", " "),
("zh", ""),
("ZH", ""),
("ja", ""),
(None, " "),
],
)
def test_asr_inter_chunk_separator_matches_protocol(language, expected_sep):
sep = asr_inter_chunk_separator(language, SupportsTranscription.no_space_languages)
assert sep == expected_sep
def test_qwen3_asr_stream_processor_passes_plain_text_without_prefix():
post_processor = (
Qwen3ASRForConditionalGeneration.get_streaming_post_processor_cls()()
)
assert post_processor.process_delta("Hello", False) == "Hello"
assert post_processor.process_delta(" world", True) == " world"
def test_qwen3_asr_stream_processor_buffers_prefix_with_leading_space():
post_processor = (
Qwen3ASRForConditionalGeneration.get_streaming_post_processor_cls()()
)
assert post_processor.process_delta(" language Eng", False) == ""
assert post_processor.process_delta("lish<asr", False) == ""
assert post_processor.process_delta("_text>Hello", True) == "Hello"
def test_qwen3_asr_stream_processor_keeps_independent_state():
processor_cls = Qwen3ASRForConditionalGeneration.get_streaming_post_processor_cls()
first_processor = processor_cls()
second_processor = processor_cls()
assert first_processor.process_delta(" language Eng", False) == ""
assert second_processor.process_delta("plain text", True) == "plain text"
assert first_processor.process_delta("lish<asr_text>Hello", True) == "Hello"
def test_qwen3_asr_stream_processor_emits_finished_incomplete_prefix():
post_processor = (
Qwen3ASRForConditionalGeneration.get_streaming_post_processor_cls()()
)
assert (
post_processor.process_delta(" language English", True) == " language English"
)
def test_qwen3_asr_stream_processor_stops_buffering_long_plain_prefix():
post_processor = (
Qwen3ASRForConditionalGeneration.get_streaming_post_processor_cls()()
)
text = " language " + ("x" * 50)
assert post_processor.process_delta(text, False) == text
def test_qwen3_asr_stream_processor_stops_buffering_prefix_with_newline():
post_processor = (
Qwen3ASRForConditionalGeneration.get_streaming_post_processor_cls()()
)
text = " language English\nhello"
assert post_processor.process_delta(text, False) == text
def test_joined_chunks_english_has_space_between():
sep = asr_inter_chunk_separator("en", SupportsTranscription.no_space_languages)
assert sep.join(["hello", "world"]) == "hello world"
def test_joined_chunks_chinese_has_no_space_between():
sep = asr_inter_chunk_separator("zh", SupportsTranscription.no_space_languages)
assert sep.join(["你好", "世界"]) == "你好世界"
# --- Integration: serving (no model) -----------------------------------------
class _StubTranscriptionModel:
"""Minimal stand-in for a SupportsTranscription implementation (no torch)."""
no_space_languages: set[str] = {"ja", "zh"}
supports_segment_timestamp = False
@classmethod
def get_speech_to_text_config(
cls, model_config: ModelConfig, task_type: str
) -> SpeechToTextConfig:
return SpeechToTextConfig(
sample_rate=16000.0,
max_audio_clip_s=5.0,
)
@classmethod
def post_process_output(cls, text: str) -> str:
return text
@classmethod
def get_streaming_post_processor_cls(
cls,
) -> type[StreamingTranscriptionPostProcessor]:
return StreamingTranscriptionPostProcessor
def _request_output(text: str, finish_reason: str | None = "stop") -> RequestOutput:
return RequestOutput(
request_id="rid",
prompt=None,
prompt_token_ids=None,
prompt_logprobs=None,
outputs=[
CompletionOutput(
index=0,
text=text,
token_ids=(1, 2, 3),
cumulative_logprob=None,
logprobs=None,
finish_reason=finish_reason,
)
],
finished=True,
)
def _sse_delta_contents(sse_body: str) -> list[str]:
"""Extract ``choices[0].delta.content`` from each ``data:`` line (streaming API)."""
contents: list[str] = []
for line in sse_body.splitlines():
if not line.startswith("data: "):
continue
payload = line.removeprefix("data: ").strip()
if payload == "[DONE]":
continue
obj = json.loads(payload)
for choice in obj.get("choices") or []:
delta = choice.get("delta") or {}
if "content" in delta:
contents.append(delta["content"])
return contents
@pytest.mark.asyncio
async def test_transcription_stream_generator_english_inserts_space_between_chunks():
"""Online streaming: first output per audio chunk is prefixed with *separator*."""
async def gen_hello() -> AsyncGenerator[RequestOutput, None]:
yield _request_output("hello")
async def gen_world() -> AsyncGenerator[RequestOutput, None]:
yield _request_output("world")
serving = OpenAIServingTranscription.__new__(OpenAIServingTranscription)
serving.enable_force_include_usage = False
serving.model_cls = _StubTranscriptionModel
serving.streaming_post_processor_cls = (
_StubTranscriptionModel.get_streaming_post_processor_cls()
)
serving.task_type = "transcribe"
request = SimpleNamespace(
model="stub-model",
stream_include_usage=False,
stream_continuous_usage_stats=False,
)
sep = asr_inter_chunk_separator("en", _StubTranscriptionModel.no_space_languages)
assert sep == " "
out_lines: list[str] = []
agen = OpenAIServingTranscription.transcription_stream_generator(
serving,
request=request,
result_generator=[gen_hello(), gen_world()],
request_id="test-req",
request_metadata=RequestResponseMetadata(request_id="test-req"),
audio_duration_s=1.0,
separator=sep,
)
async for line in agen:
out_lines.append(line)
sse = "".join(out_lines)
combined = "".join(_sse_delta_contents(sse))
assert combined.strip() == "hello world"
@pytest.mark.asyncio
async def test_transcription_stream_generator_chinese_no_space_between_chunks():
async def gen_a() -> AsyncGenerator[RequestOutput, None]:
yield _request_output("你好")
async def gen_b() -> AsyncGenerator[RequestOutput, None]:
yield _request_output("世界")
serving = OpenAIServingTranscription.__new__(OpenAIServingTranscription)
serving.enable_force_include_usage = False
serving.model_cls = _StubTranscriptionModel
serving.streaming_post_processor_cls = (
_StubTranscriptionModel.get_streaming_post_processor_cls()
)
serving.task_type = "transcribe"
request = SimpleNamespace(
model="stub-model",
stream_include_usage=False,
stream_continuous_usage_stats=False,
)
sep = asr_inter_chunk_separator("zh", _StubTranscriptionModel.no_space_languages)
assert sep == ""
out_lines: list[str] = []
agen = OpenAIServingTranscription.transcription_stream_generator(
serving,
request=request,
result_generator=[gen_a(), gen_b()],
request_id="test-req-zh",
request_metadata=RequestResponseMetadata(request_id="test-req-zh"),
audio_duration_s=1.0,
separator=sep,
)
async for line in agen:
out_lines.append(line)
combined = "".join(_sse_delta_contents("".join(out_lines)))
assert combined == "你好世界"
@pytest.mark.asyncio
async def test_transcription_stream_generator_strips_qwen3_asr_prefix_per_chunk():
async def gen_hello() -> AsyncGenerator[RequestOutput, None]:
yield _request_output("language Eng", finish_reason=None)
yield _request_output("lish<asr", finish_reason=None)
yield _request_output("_text>Hello", finish_reason=None)
yield _request_output("")
async def gen_world() -> AsyncGenerator[RequestOutput, None]:
yield _request_output(" language Eng", finish_reason=None)
yield _request_output("lish<asr_text>world")
serving = OpenAIServingTranscription.__new__(OpenAIServingTranscription)
serving.enable_force_include_usage = False
serving.model_cls = Qwen3ASRForConditionalGeneration
serving.streaming_post_processor_cls = (
Qwen3ASRForConditionalGeneration.get_streaming_post_processor_cls()
)
serving.task_type = "transcribe"
request = SimpleNamespace(
model="stub-qwen3-asr",
stream_include_usage=False,
stream_continuous_usage_stats=False,
)
out_lines: list[str] = []
agen = OpenAIServingTranscription.transcription_stream_generator(
serving,
request=request,
result_generator=[gen_hello(), gen_world()],
request_id="test-qwen3-asr",
request_metadata=RequestResponseMetadata(request_id="test-qwen3-asr"),
audio_duration_s=1.0,
separator=" ",
)
async for line in agen:
out_lines.append(line)
combined = "".join(_sse_delta_contents("".join(out_lines)))
assert combined == "Hello world"
@pytest.mark.asyncio
async def test_create_transcription_non_streaming_joins_chunks_by_language():
"""``create_transcription`` uses the same separator logic as the helper."""
async def gen_hello() -> AsyncGenerator[RequestOutput, None]:
yield _request_output("hello")
async def gen_world() -> AsyncGenerator[RequestOutput, None]:
yield _request_output("world")
engine_client = MagicMock()
engine_client.model_config = MagicMock()
engine_client.model_config.get_diff_sampling_param.return_value = {
"max_tokens": 256,
"temperature": 0.0,
}
engine_client.model_config.max_model_len = 8192
engine_client.errored = False
engine_client.generate.side_effect = [gen_hello(), gen_world()]
models = MagicMock(spec=OpenAIServingModels)
models.lora_requests = {}
models.is_base_model.return_value = True
preprocess_mock = AsyncMock(
return_value=([MagicMock(), MagicMock()], 1.0, [0.0, 29.5])
)
with (
patch(
"vllm.model_executor.model_loader.get_model_cls",
return_value=_StubTranscriptionModel,
),
patch.object(
SpeechToTextBaseServing, "_preprocess_speech_to_text", preprocess_mock
),
):
serving = OpenAIServingTranscription(engine_client, models, request_logger=None)
req_en = TranscriptionRequest.model_construct(
file=MagicMock(),
model="stub-model",
language="en",
stream=False,
response_format="json",
)
out_en = await serving.create_transcription(
b"\x00\x00", req_en, raw_request=None
)
assert not isinstance(out_en, ErrorResponse)
assert out_en.text == "hello world"
async def gen_nihao() -> AsyncGenerator[RequestOutput, None]:
yield _request_output("你好")
async def gen_shijie() -> AsyncGenerator[RequestOutput, None]:
yield _request_output("世界")
engine_client.generate.side_effect = [gen_nihao(), gen_shijie()]
req_zh = TranscriptionRequest.model_construct(
file=MagicMock(),
model="stub-model",
language="zh",
stream=False,
response_format="json",
)
out_zh = await serving.create_transcription(
b"\x00\x00", req_zh, raw_request=None
)
assert not isinstance(out_zh, ErrorResponse)
assert out_zh.text == "你好世界"
@@ -1,130 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Non-object JSON bodies must fail validation cleanly (4xx), not AttributeError (500).
mode=before validators that call data.get(...) without an isinstance(data, dict)
guard raise AttributeError for string/list/scalar bodies and surface as HTTP 500.
This extends the chat completion coverage added in #51654 to the remaining
request models whose before-validators were missing the same guard.
"""
import pytest
from pydantic import ValidationError
from vllm.entrypoints.generate.generative_scoring.serving import (
GenerativeScoringRequest,
)
from vllm.entrypoints.generate.structured_decisions.protocol import (
StructuredDecisionRequest,
)
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.entrypoints.pooling.classify.protocol import ClassificationChatRequest
from vllm.entrypoints.pooling.embed.protocol import EmbeddingChatRequest
from vllm.entrypoints.pooling.pooling.protocol import PoolingChatRequest
from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateRequest
from vllm.entrypoints.serve.tokenize.protocol import TokenizeChatRequest
from vllm.entrypoints.speech_to_text.transcription.protocol import TranscriptionRequest
from vllm.entrypoints.speech_to_text.translation.protocol import TranslationRequest
from vllm.exceptions import VLLMValidationError
pytestmark = pytest.mark.skip_global_cleanup
REQUEST_MODELS = [
GenerativeScoringRequest,
CompletionRequest,
ResponsesRequest,
EmbeddingChatRequest,
ClassificationChatRequest,
PoolingChatRequest,
TokenizeChatRequest,
TranscriptionRequest,
TranslationRequest,
]
@pytest.mark.parametrize("request_model", REQUEST_MODELS, ids=lambda m: m.__name__)
@pytest.mark.parametrize(
"payload",
[
"this is not valid json{{{",
["not", "an", "object"],
42,
None,
True,
],
)
def test_request_models_reject_non_object_body(request_model, payload):
with pytest.raises(ValidationError):
request_model.model_validate(payload)
def test_completion_request_still_validates_dict_bodies():
"""The guard must not swallow real field-level errors on object bodies."""
with pytest.raises(VLLMValidationError, match="prompt"):
CompletionRequest.model_validate({"model": "qwen", "prompt": ""})
def test_tokenize_chat_request_still_validates_dict_bodies():
with pytest.raises(VLLMValidationError, match="add_generation_prompt"):
TokenizeChatRequest.model_validate(
{
"model": "qwen",
"messages": [{"role": "user", "content": "hello"}],
"continue_final_message": True,
"add_generation_prompt": True,
}
)
CACHE_SALT_MODELS = [
(
GenerativeScoringRequest,
{
"query": "Is this city the capital of France?",
"items": ["Paris"],
"label_token_ids": [9454, 2753],
},
),
(
StructuredDecisionRequest,
{
"state": "hello",
"questions": {
"lang": {
"type": "choice",
"criteria": {"English": "Text written in English."},
}
},
},
),
(CompletionRequest, {"prompt": "hello"}),
(ChatCompletionRequest, {"messages": [{"role": "user", "content": "hello"}]}),
(ResponsesRequest, {"input": "hello"}),
(GenerateRequest, {"token_ids": [1], "sampling_params": {}}),
]
@pytest.mark.parametrize(
("request_model", "payload"),
CACHE_SALT_MODELS,
ids=[m.__name__ for m, _ in CACHE_SALT_MODELS],
)
def test_request_models_reject_forbidden_cache_salt(request_model, payload):
with pytest.raises(VLLMValidationError, match="cache_salt"):
request_model.model_validate({**payload, "cache_salt": "/"})
def test_request_models_reject_overlong_cache_salt():
with pytest.raises(VLLMValidationError, match="cache_salt"):
CompletionRequest.model_validate({"prompt": "hello", "cache_salt": "a" * 129})
def test_request_models_accept_safe_cache_salt():
request = CompletionRequest.model_validate(
{"prompt": "hello", "cache_salt": "safe-cache_salt-123"}
)
assert request.cache_salt == "safe-cache_salt-123"
-471
View File
@@ -1,471 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Correctness tests for the AMX-only CPU MLA backend: the vendored
decode/extend/bmm kernels, the KV cache write, and the ``AMXMLAImpl``
backend built on top of them.
"""
import pytest
import torch
from vllm.platforms import current_platform
if not current_platform.is_cpu():
pytest.skip("skipping CPU-only tests", allow_module_level=True)
if not torch.cpu._is_amx_tile_supported():
pytest.skip("AMX MLA requires an AMX-capable host", allow_module_level=True)
torch.cpu._init_amx()
from vllm import _custom_ops as ops # noqa: E402
from vllm.utils.torch_utils import set_random_seed # noqa: E402
from vllm.v1.attention.backends.mla.amx_mla import ( # noqa: E402
AMXMLAImpl,
_compute_num_kv_splits,
_expand_block_table,
)
KV_LORA_RANK = 64
QK_ROPE_HEAD_DIM = 32
HEAD_SIZE = KV_LORA_RANK + QK_ROPE_HEAD_DIM # 96, kv-cache row width
# The extend/decode kernels transpose by 512-bit lanes: both the cache row
# width and the (kv_lora_rank-wide) value width must be 32-element aligned.
DTYPE = torch.bfloat16
ATOL = 2e-2
RTOL = 2e-2
# Backend-level tests chain absorb -> attend -> de-absorb through bf16,
# accumulating more rounding error than a single kernel op.
_IMPL_ATOL = 1.5e-1
_IMPL_RTOL = 1.5e-1
def _flatten_cache(kv_cache: torch.Tensor) -> torch.Tensor:
"""(num_blocks, block_size, head_size) -> (num_slots, head_size)."""
return kv_cache.view(-1, kv_cache.size(-1))
def _random_paged_cache(num_slots: int, head_size: int) -> torch.Tensor:
return torch.randn(num_slots, head_size, dtype=DTYPE)
def _ref_latent_attn(
q: torch.Tensor,
keys: torch.Tensor,
scale: float,
kv_lora_rank: int,
) -> torch.Tensor:
"""Plain-PyTorch causal-free latent-space MQA attention for one request.
q: (num_heads, head_size); keys: (ctx_len, head_size). Values are the
first ``kv_lora_rank`` columns of the same latent rows (MLA's K/V
aliasing), matching what decode_attention_cpu/extend_attention_cpu
compute internally for the MLA-shaped case.
"""
values = keys[:, :kv_lora_rank]
scores = (q.float() @ keys.float().T) * scale
probs = torch.softmax(scores, dim=-1)
return (probs @ values.float()).to(q.dtype)
def test_bmm_cpu_matches_torch_bmm():
set_random_seed(0)
# bmm_cpu requires the output's last dim (mat2's out-features) to be a
# multiple of 32 (tinygemm's tile width); the contraction dim is free.
n, b, p, l = 8, 32, 32, 32 # noqa: E741
mat1 = torch.randn(n, b, p, dtype=DTYPE)
mat2 = torch.randn(n, l, p, dtype=DTYPE) # (N, OUT, IN) Linear convention
ref = torch.bmm(mat1.float(), mat2.float().transpose(1, 2)).to(DTYPE)
out = torch.empty(n, b, l, dtype=DTYPE)
ops.bmm_cpu(out, mat1, mat2, False, None)
torch.testing.assert_close(out, ref, atol=ATOL, rtol=RTOL)
packed = torch.ops._C.convert_weight_packed(mat2.contiguous())
out_vnni = torch.empty(n, b, l, dtype=DTYPE)
ops.bmm_cpu(out_vnni, mat1, packed, True, None)
torch.testing.assert_close(out_vnni, ref, atol=ATOL, rtol=RTOL)
def test_concat_and_cache_mla_round_trip():
set_random_seed(0)
num_tokens = 37
num_blocks, block_size = 8, 16
kv_cache = torch.zeros(num_blocks, block_size, HEAD_SIZE, dtype=DTYPE)
kv_c_normed = torch.randn(num_tokens, KV_LORA_RANK, dtype=DTYPE)
k_pe = torch.randn(num_tokens, QK_ROPE_HEAD_DIM, dtype=DTYPE)
slot_mapping = torch.randperm(num_blocks * block_size)[:num_tokens].to(torch.int64)
ops.amx_mla_concat_and_cache(kv_c_normed, k_pe, kv_cache, slot_mapping)
flat = _flatten_cache(kv_cache)
read_back = flat[slot_mapping]
expected = torch.cat([kv_c_normed, k_pe], dim=-1)
torch.testing.assert_close(read_back, expected)
untouched = torch.ones(num_blocks * block_size, dtype=torch.bool)
untouched[slot_mapping] = False
assert (flat[untouched] == 0).all()
@pytest.mark.parametrize("seq_lens", [[5], [1, 300, 33], [513, 1, 129, 7]])
def test_decode_attention_cpu_matches_reference(seq_lens):
set_random_seed(0)
num_seqs = len(seq_lens)
num_heads = 4
block_size = 32
max_len = max(seq_lens)
max_blocks = (max_len + block_size - 1) // block_size
num_blocks = num_seqs * max_blocks
kv_cache = _random_paged_cache(num_blocks * block_size, HEAD_SIZE)
block_table = torch.randperm(num_blocks).view(num_seqs, max_blocks).to(torch.int64)
req_to_token = _expand_block_table(block_table, block_size)
req_pool_indices = torch.arange(num_seqs, dtype=torch.int64)
seq_lens_t = torch.tensor(seq_lens, dtype=torch.int64)
scale = HEAD_SIZE**-0.5
q = torch.randn(num_seqs, num_heads, HEAD_SIZE, dtype=DTYPE)
kv_cache_flat = kv_cache.view(-1, 1, HEAD_SIZE)
v_buffer = kv_cache_flat[..., :KV_LORA_RANK]
o = torch.zeros(num_seqs, num_heads, KV_LORA_RANK, dtype=DTYPE)
attn_logits = torch.zeros(
num_seqs, num_heads, 4, KV_LORA_RANK + 1, dtype=torch.float32
)
ops.cpu_mla_decode(
q,
kv_cache_flat,
v_buffer,
o,
None,
None,
None,
attn_logits,
req_to_token,
req_pool_indices,
seq_lens_t,
scale,
0.0,
False,
0,
None,
None,
)
flat_cache = _flatten_cache(kv_cache)
for i, seq_len in enumerate(seq_lens):
token_ids = req_to_token[i, :seq_len]
keys = flat_cache[token_ids]
ref = _ref_latent_attn(q[i], keys, scale, KV_LORA_RANK)
torch.testing.assert_close(o[i], ref, atol=ATOL, rtol=RTOL)
@pytest.mark.parametrize(
"prefix_lens,extend_lens",
[
([0], [9]), # fresh prefill, no cached prefix
([64], [17]), # continuation of a cached prefix
([0, 40, 128], [23, 5, 31]), # mixed batch
],
)
def test_extend_attention_cpu_matches_reference(prefix_lens, extend_lens):
set_random_seed(0)
num_seqs = len(prefix_lens)
num_heads = 4
block_size = 32
seq_lens = [p + e for p, e in zip(prefix_lens, extend_lens)]
max_len = max(seq_lens)
max_blocks = (max_len + block_size - 1) // block_size
num_blocks = num_seqs * max_blocks
kv_cache = _random_paged_cache(num_blocks * block_size, HEAD_SIZE)
block_table = torch.randperm(num_blocks).view(num_seqs, max_blocks).to(torch.int64)
req_to_token = _expand_block_table(block_table, block_size)
req_pool_indices = torch.arange(num_seqs, dtype=torch.int64)
seq_lens_t = torch.tensor(seq_lens, dtype=torch.int64)
extend_seq_lens_t = torch.tensor(extend_lens, dtype=torch.int64)
extend_start_loc = torch.cumsum(
torch.tensor([0, *extend_lens[:-1]], dtype=torch.int64), dim=0
)
scale = HEAD_SIZE**-0.5
total_new_tokens = sum(extend_lens)
# The new tokens' latent K/V, already written into the cache (mirrors
# do_kv_cache_update running before forward_mha in real usage).
k_extend_flat = torch.randn(total_new_tokens, HEAD_SIZE, dtype=DTYPE)
flat_cache = _flatten_cache(kv_cache)
for i in range(num_seqs):
prefix_len = prefix_lens[i]
ext_len = extend_lens[i]
new_token_ids = req_to_token[i, prefix_len : prefix_len + ext_len]
start = extend_start_loc[i].item()
flat_cache[new_token_ids] = k_extend_flat[start : start + ext_len]
q_extend = torch.randn(total_new_tokens, num_heads, HEAD_SIZE, dtype=DTYPE)
k_extend = k_extend_flat.unsqueeze(1)
v_extend = k_extend[..., :KV_LORA_RANK]
kv_cache_flat = kv_cache.view(-1, 1, HEAD_SIZE)
v_buffer = kv_cache_flat[..., :KV_LORA_RANK]
o_extend = torch.empty(total_new_tokens, num_heads, KV_LORA_RANK, dtype=DTYPE)
ops.cpu_mla_extend(
q_extend,
k_extend,
v_extend,
o_extend,
kv_cache_flat,
v_buffer,
req_to_token,
req_pool_indices,
seq_lens_t,
extend_seq_lens_t,
extend_start_loc,
max(extend_lens),
scale,
0.0,
False,
0,
None,
None,
None,
)
for i in range(num_seqs):
prefix_len = prefix_lens[i]
ext_len = extend_lens[i]
start = extend_start_loc[i].item()
for t in range(ext_len):
visible = prefix_len + t + 1
token_ids = req_to_token[i, :visible]
keys = flat_cache[token_ids]
ref = _ref_latent_attn(q_extend[start + t], keys, scale, KV_LORA_RANK)
torch.testing.assert_close(o_extend[start + t], ref, atol=ATOL, rtol=RTOL)
class _FakeLinear:
"""Minimal duck-typed stand-in for kv_b_proj: only .weight and
.quant_method are read by get_and_maybe_dequant_weights()."""
def __init__(self, weight: torch.Tensor):
self.weight = weight
self.quant_method = None
def _make_amx_mla_impl(num_heads, qk_nope_head_dim, v_head_dim, kv_lora_rank):
weight = torch.randn(
num_heads * (qk_nope_head_dim + v_head_dim), kv_lora_rank, dtype=DTYPE
)
kv_b_proj = _FakeLinear(weight)
impl = AMXMLAImpl(
num_heads=num_heads,
head_size=kv_lora_rank + QK_ROPE_HEAD_DIM,
scale=(qk_nope_head_dim + QK_ROPE_HEAD_DIM) ** -0.5,
num_kv_heads=1,
alibi_slopes=None,
sliding_window=None,
kv_cache_dtype="auto",
logits_soft_cap=None,
attn_type="decoder",
kv_sharing_target_layer_name=None,
q_lora_rank=None,
kv_lora_rank=kv_lora_rank,
qk_nope_head_dim=qk_nope_head_dim,
qk_rope_head_dim=QK_ROPE_HEAD_DIM,
qk_head_dim=qk_nope_head_dim + QK_ROPE_HEAD_DIM,
v_head_dim=v_head_dim,
kv_b_proj=kv_b_proj,
)
impl.process_weights_after_loading(DTYPE)
return impl, weight
class _FakePrefillMetadata:
def __init__(
self, block_table, cpu_seq_lens, query_start_loc, req_to_token, req_pool_indices
):
self.block_table = block_table
self.cpu_seq_lens = cpu_seq_lens
query_start_loc_i64 = query_start_loc.to(torch.int64)
extend_seq_lens = query_start_loc_i64[1:] - query_start_loc_i64[:-1]
self.extend_seq_lens = extend_seq_lens
self.extend_start_loc = query_start_loc_i64[:-1]
self.max_len_extend = int(extend_seq_lens.max().item())
self.req_to_token = req_to_token
self.req_pool_indices = req_pool_indices
class _FakeDecodeMetadata:
def __init__(self, block_table, seq_lens, req_to_token, req_pool_indices):
self.block_table = block_table
self.seq_lens_i64 = seq_lens.to(torch.int64)
self.req_to_token = req_to_token
self.req_pool_indices = req_pool_indices
self.num_kv_splits = _compute_num_kv_splits(
int(seq_lens.max().item()), current_platform.num_compute_units()
)
class _FakeAttnMetadata:
def __init__(self, decode=None, prefill=None, max_seq_len=0):
self.decode = decode
self.prefill = prefill
self.max_seq_len = max_seq_len
def test_amx_mla_impl_forward_mqa_matches_reference(default_vllm_config):
"""forward_mqa receives already-absorbed Q (as MLAAttention.forward_impl
would produce via layer.W_UK_T) and must reproduce plain per-head MLA
decode attention through W_UK/W_UV."""
set_random_seed(1)
num_heads, qk_nope_head_dim, v_head_dim, kv_lora_rank = 4, 32, 32, 64
impl, kv_b_weight = _make_amx_mla_impl(
num_heads, qk_nope_head_dim, v_head_dim, kv_lora_rank
)
w_uk, w_uv = kv_b_weight.T.view(
kv_lora_rank, num_heads, qk_nope_head_dim + v_head_dim
).split([qk_nope_head_dim, v_head_dim], dim=-1)
seq_lens = [1, 40, 5]
num_seqs = len(seq_lens)
block_size = 32
max_blocks = (max(seq_lens) + block_size - 1) // block_size
num_blocks = num_seqs * max_blocks
head_size = kv_lora_rank + QK_ROPE_HEAD_DIM
kv_cache = torch.randn(num_blocks, block_size, head_size, dtype=DTYPE)
block_table = torch.randperm(num_blocks).view(num_seqs, max_blocks).to(torch.int64)
req_to_token = _expand_block_table(block_table, block_size)
q_nope = torch.randn(num_seqs, num_heads, qk_nope_head_dim, dtype=DTYPE)
q_pe = torch.randn(num_seqs, num_heads, QK_ROPE_HEAD_DIM, dtype=DTYPE)
# Pre-absorb, mirroring MLAAttention.forward_impl's own bmm against W_UK_T.
ql_nope = torch.einsum("bnp,lnp->bnl", q_nope.float(), w_uk.float()).to(DTYPE)
attn_metadata = _FakeAttnMetadata(
decode=_FakeDecodeMetadata(
block_table=block_table,
seq_lens=torch.tensor(seq_lens, dtype=torch.int64),
req_to_token=req_to_token,
req_pool_indices=torch.arange(num_seqs, dtype=torch.int64),
),
max_seq_len=max(seq_lens),
)
o, lse = impl.forward_mqa((ql_nope, q_pe), kv_cache, attn_metadata, layer=None)
assert lse is None
# De-absorb, mirroring MLAAttention.forward_impl's own bmm against
# layer._v_up_proj, to get back into real per-head V space for
# comparison against a from-scratch reference.
out_real = torch.einsum("bnl,lnv->bnv", o.float(), w_uv.float())
flat_cache = _flatten_cache(kv_cache)
scale = impl.scale
for i, seq_len in enumerate(seq_lens):
token_ids = req_to_token[i, :seq_len]
keys_latent = flat_cache[token_ids] # (L, head_size)
k_nope_latent, k_pe = keys_latent.split(
[kv_lora_rank, QK_ROPE_HEAD_DIM], dim=-1
)
k_real = torch.einsum("lp,pnd->nld", k_nope_latent.float(), w_uk.float())
k_pe_b = k_pe.float().unsqueeze(0).expand(num_heads, -1, -1)
k_full = torch.cat([k_real, k_pe_b], dim=-1) # (num_heads, L, qk_head_dim)
q_full = torch.cat(
[q_nope[i], q_pe[i]], dim=-1
).float() # (num_heads, qk_head_dim)
scores = torch.einsum("nd,nld->nl", q_full, k_full) * scale
probs = torch.softmax(scores, dim=-1)
v_real = torch.einsum("lp,pnv->nlv", k_nope_latent.float(), w_uv.float())
ref = torch.einsum("nl,nlv->nv", probs, v_real)
torch.testing.assert_close(out_real[i], ref, atol=_IMPL_ATOL, rtol=_IMPL_RTOL)
def test_amx_mla_impl_forward_mha_matches_reference(default_vllm_config):
"""forward_mha receives raw unabsorbed Q and must attend correctly for a
prefill batch with a mix of fresh and continued (cached-prefix)
sequences."""
set_random_seed(2)
num_heads, qk_nope_head_dim, v_head_dim, kv_lora_rank = 4, 32, 32, 64
impl, kv_b_weight = _make_amx_mla_impl(
num_heads, qk_nope_head_dim, v_head_dim, kv_lora_rank
)
w_uk, w_uv = kv_b_weight.T.view(
kv_lora_rank, num_heads, qk_nope_head_dim + v_head_dim
).split([qk_nope_head_dim, v_head_dim], dim=-1)
prefix_lens = [0, 20]
extend_lens = [13, 7]
seq_lens = [p + e for p, e in zip(prefix_lens, extend_lens)]
num_seqs = len(prefix_lens)
block_size = 32
max_blocks = (max(seq_lens) + block_size - 1) // block_size
num_blocks = num_seqs * max_blocks
head_size = kv_lora_rank + QK_ROPE_HEAD_DIM
kv_cache = torch.randn(num_blocks, block_size, head_size, dtype=DTYPE)
block_table = torch.randperm(num_blocks).view(num_seqs, max_blocks).to(torch.int64)
req_to_token = _expand_block_table(block_table, block_size)
flat_cache = _flatten_cache(kv_cache)
total_new_tokens = sum(extend_lens)
query_start_loc = torch.tensor(
[0, *torch.cumsum(torch.tensor(extend_lens), dim=0).tolist()],
dtype=torch.int64,
)
q_nope = torch.randn(total_new_tokens, num_heads, qk_nope_head_dim, dtype=DTYPE)
q_pe = torch.randn(total_new_tokens, num_heads, QK_ROPE_HEAD_DIM, dtype=DTYPE)
q = torch.cat([q_nope, q_pe], dim=-1)
kv_c_normed = torch.randn(total_new_tokens, kv_lora_rank, dtype=DTYPE)
k_pe = torch.randn(total_new_tokens, 1, QK_ROPE_HEAD_DIM, dtype=DTYPE)
# Write the new tokens' latent K/V into the cache ahead of time, mirroring
# do_kv_cache_update running before forward_mha in real usage.
for i in range(num_seqs):
prefix_len = prefix_lens[i]
ext_len = extend_lens[i]
new_token_ids = req_to_token[i, prefix_len : prefix_len + ext_len]
start = query_start_loc[i].item()
flat_cache[new_token_ids, :kv_lora_rank] = kv_c_normed[start : start + ext_len]
flat_cache[new_token_ids, kv_lora_rank:] = k_pe[start : start + ext_len, 0]
attn_metadata = _FakeAttnMetadata(
prefill=_FakePrefillMetadata(
block_table=block_table,
cpu_seq_lens=torch.tensor(seq_lens, dtype=torch.int64),
query_start_loc=query_start_loc,
req_to_token=req_to_token,
req_pool_indices=torch.arange(num_seqs, dtype=torch.int64),
)
)
output = torch.empty(total_new_tokens, num_heads * v_head_dim, dtype=DTYPE)
impl.forward_mha(
q, kv_c_normed, k_pe, kv_cache, attn_metadata, k_scale=None, output=output
)
out_view = output.view(total_new_tokens, num_heads, v_head_dim).float()
scale = impl.scale
for i in range(num_seqs):
prefix_len = prefix_lens[i]
ext_len = extend_lens[i]
start = query_start_loc[i].item()
for t in range(ext_len):
visible = prefix_len + t + 1
token_ids = req_to_token[i, :visible]
keys_latent = flat_cache[token_ids]
k_nope_latent, k_pe_ref = keys_latent.split(
[kv_lora_rank, QK_ROPE_HEAD_DIM], dim=-1
)
k_real = torch.einsum("lp,pnd->nld", k_nope_latent.float(), w_uk.float())
k_pe_b = k_pe_ref.float().unsqueeze(0).expand(num_heads, -1, -1)
k_full = torch.cat([k_real, k_pe_b], dim=-1)
q_full = torch.cat([q_nope[start + t], q_pe[start + t]], dim=-1).float()
scores = torch.einsum("nd,nld->nl", q_full, k_full) * scale
probs = torch.softmax(scores, dim=-1)
v_real = torch.einsum("lp,pnv->nlv", k_nope_latent.float(), w_uv.float())
ref = torch.einsum("nl,nlv->nv", probs, v_real)
torch.testing.assert_close(
out_view[start + t], ref, atol=_IMPL_ATOL, rtol=_IMPL_RTOL
)
@@ -1,919 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from contextlib import contextmanager
from typing import Literal
from unittest.mock import MagicMock, patch
import pytest
import torch
from vllm.config import (
AttentionConfig,
CacheConfig,
ParallelConfig,
VllmConfig,
set_current_vllm_config,
)
from vllm.config.cache import CacheDType
from vllm.platforms import current_platform
from vllm.platforms.cpu import CpuPlatform
from vllm.platforms.interface import DeviceCapability
if current_platform.is_cuda():
from vllm.platforms.cuda import CudaPlatform
else:
CudaPlatform = None
if current_platform.is_rocm():
from vllm.platforms.rocm import RocmPlatform
else:
RocmPlatform = None # type: ignore[misc] # Unavailable platform import.
from vllm.v1.attention.backend import AttentionType
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.attention.selector import (
AttentionSelectorConfig,
_cached_get_attn_backend,
get_attn_backend,
)
@pytest.fixture(autouse=True)
def clear_cache():
"""Clear lru cache to ensure each test case runs without caching."""
_cached_get_attn_backend.cache_clear()
# Define MLA and non-MLA backends separately
DEVICE_MLA_BACKENDS = {
"cuda": [
"TRITON_MLA",
"FLASHMLA",
"FLASHINFER_MLA",
"FLASH_ATTN_MLA",
"CUTLASS_MLA",
],
"hip": ["TRITON_MLA", "ROCM_AITER_MLA"],
"cpu": [],
}
DEVICE_REGULAR_ATTN_BACKENDS = {
"cuda": ["FLASHINFER", "FLASH_ATTN"],
"hip": ["ROCM_ATTN"],
"cpu": ["CPU_ATTN"],
}
DEVICE_MLA_BLOCK_SIZES = {
"cuda": [16, 64], # CUDA supports both standard and extended block sizes
"hip": [16, 1], # HIP requires special handling for block_size=1
# "cpu": [16] # CPU uses fixed block size from test cases
"cpu": [], # FIXME(woosuk): Temporarily disable CPU tests
}
def generate_params():
is_rocm = current_platform.is_rocm()
params = []
device_list = ["cuda", "cpu"] if not is_rocm else ["hip", "cpu"]
for use_mla in [True, False]:
for device in device_list:
backends = (
DEVICE_MLA_BACKENDS[device]
if use_mla
else DEVICE_REGULAR_ATTN_BACKENDS[device]
)
for name in backends:
block_sizes = (
[128]
if name == "CUTLASS_MLA"
else DEVICE_MLA_BLOCK_SIZES[device]
if use_mla
else [16]
)
for block_size in block_sizes:
for use_dcp in [False, True] if device != "cpu" else [False]:
params.append(
pytest.param(
device,
name,
use_mla,
block_size,
use_dcp,
id=(
f"{device}_{name}_mla_{str(use_mla)[0]}"
f"_blks{block_size}_dcp{use_dcp}"
),
)
)
return params
@pytest.mark.parametrize(
"device, name, use_mla, block_size, use_dcp", generate_params()
)
def test_backend_selection(
device: str,
name: str,
use_mla: bool,
block_size: int,
use_dcp: bool,
):
"""Supported GPU backends remain selectable when DCP is enabled."""
# Create AttentionConfig with the specified backend
attention_config = AttentionConfig(backend=AttentionBackendEnum[name])
cache_config = CacheConfig(block_size=block_size)
vllm_config = VllmConfig(
attention_config=attention_config,
cache_config=cache_config,
parallel_config=ParallelConfig(
# Skip GPU-count autodetection: selection does not launch workers.
distributed_executor_backend="mp" if use_dcp else None,
tensor_parallel_size=2 if use_dcp else 1,
decode_context_parallel_size=2 if use_dcp else 1,
),
)
with set_current_vllm_config(vllm_config):
if device == "cpu":
with patch("vllm.platforms.current_platform", CpuPlatform()):
backend = get_attn_backend(16, torch.float16, None)
assert backend.get_name() == "CPU_ATTN"
elif device == "hip":
if RocmPlatform is None:
pytest.skip("RocmPlatform not available")
with patch("vllm.platforms.current_platform", RocmPlatform()):
if use_mla:
# ROCm MLA backend logic:
# - TRITON_MLA: supported when block_size != 1
# - ROCM_AITER_MLA: supported when block_size == 1
# If backend is forced but doesn't match block_size,
# should raise ValueError
if name == "TRITON_MLA" and block_size == 1:
# TRITON_MLA doesn't support block_size == 1
with pytest.raises(ValueError):
get_attn_backend(576, torch.float16, None, use_mla=use_mla)
else:
# Valid backend-block_size combination
backend = get_attn_backend(
576, torch.float16, None, use_mla=use_mla
)
expected = name
assert backend.get_name() == expected
else:
if use_dcp:
with pytest.raises(ValueError, match="DCP not supported"):
get_attn_backend(32, torch.float16, None, use_mla=use_mla)
else:
backend = get_attn_backend(
32, torch.float16, None, use_mla=use_mla
)
expected = "ROCM_ATTN"
assert backend.get_name() == expected
elif device == "cuda":
if CudaPlatform is None:
pytest.skip("CudaPlatform not available")
with patch("vllm.platforms.current_platform", CudaPlatform()):
capability = torch.cuda.get_device_capability()
if use_mla:
# CUDA MLA backend logic:
# - CUTLASS_MLA: only supported with block_size == 128
# and Blackwell GPUs (SM 10.x), V1 only
# - FLASHINFER_MLA: only supported on Blackwell GPUs
# (SM 10.x), V1 only
# - FLASHMLA: only supported with block_size == 64
# - FLASH_ATTN_MLA: V1 only
# - TRITON_MLA: fallback for other cases
if name == "CUTLASS_MLA":
if block_size != 128:
# CUTLASS_MLA only supports block_size == 128
pytest.skip("CUTLASS_MLA only supports block_size 128")
if capability[0] != 10:
pytest.skip("CUTLASS MLA is not supported on this platform")
backend = get_attn_backend(
576, torch.float16, None, use_mla=use_mla
)
expected = "CUTLASS_MLA"
assert backend.get_name() == expected
elif name == "FLASHINFER_MLA":
if capability[0] != 10:
pytest.skip(
"FlashInfer MLA is not supported on this platform"
)
if block_size not in [32, 64]:
# FlashInfer MLA only supports block_size 32 or 64
pytest.skip(
"FlashInfer MLA only supports block_size 32 or 64"
)
backend = get_attn_backend(
576, torch.float16, None, use_mla=use_mla
)
expected = "FLASHINFER_MLA"
assert backend.get_name() == expected
elif name == "FLASHMLA":
if block_size != 64:
# FlashMLA only supports block_size == 64
pytest.skip("FlashMLA only supports block_size 64")
from vllm.v1.attention.backends.mla.flashmla import (
is_flashmla_dense_supported,
)
is_supported, _ = is_flashmla_dense_supported()
if not is_supported:
pytest.skip("FlashMLA not supported on this platform")
backend = get_attn_backend(
576,
torch.float16,
None,
use_mla=use_mla,
)
expected = name
assert backend.get_name() == expected
elif name == "FLASH_ATTN_MLA":
from vllm.v1.attention.backends.fa_utils import (
flash_attn_supports_mla,
)
if not flash_attn_supports_mla():
pytest.skip(
"FlashAttention MLA not supported on this platform"
)
backend = get_attn_backend(
576, torch.float16, None, use_mla=use_mla
)
expected = "FLASH_ATTN_MLA"
assert backend.get_name() == expected
else:
# TRITON_MLA or other fallback
backend = get_attn_backend(
576, torch.float16, None, use_mla=use_mla
)
expected = "TRITON_MLA"
assert backend.get_name() == expected
elif name == "FLASHINFER":
backend = get_attn_backend(64, torch.float16, None, use_mla=use_mla)
expected = "FLASHINFER"
assert backend.get_name() == expected
elif name == "FLASH_ATTN":
backend = get_attn_backend(32, torch.float16, None, use_mla=use_mla)
expected = "FLASH_ATTN"
assert backend.get_name() == expected
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="GPU attention backends"
)
@pytest.mark.parametrize(
"backend_name",
[
"FLASH_ATTN",
"FLASH_ATTN_DIFFKV",
"FLASHINFER",
"TRITON_MLA",
"ROCM_AITER_MLA",
"ROCM_AITER_TRITON_MLA",
"CUTLASS_MLA",
"FLASHMLA",
"FLASHINFER_MLA",
"TOKENSPEED_MLA",
"FLASH_ATTN_MLA",
"FLASHMLA_SPARSE",
"FLASHINFER_MLA_SPARSE",
],
)
def test_supported_backend_preserves_dcp_eligibility(backend_name, default_vllm_config):
"""DCP opt-ins preserve eligibility and unrelated configuration restrictions."""
if backend_name.startswith("ROCM_") and not current_platform.is_rocm():
pytest.skip("ROCm-specific backend")
try:
backend_cls = AttentionBackendEnum[backend_name].get_class()
except ImportError as error:
pytest.skip(f"Optional backend dependency unavailable: {error}")
config = AttentionSelectorConfig(
head_size=576 if backend_cls.is_mla() else 128,
dtype=torch.bfloat16,
kv_cache_dtype="auto",
block_size=backend_cls.get_preferred_block_size(64),
use_mla=backend_cls.is_mla(),
use_sparse=backend_cls.is_sparse(),
)
kwargs = dict(
device_capability=current_platform.get_device_capability(), **config._asdict()
)
invalid_without_dcp = backend_cls.validate_configuration(**kwargs)
kwargs["use_dcp"] = True
assert backend_cls.validate_configuration(**kwargs) == invalid_without_dcp
@pytest.mark.parametrize("device", ["cpu", "cuda", "hip"])
def test_fp32_fallback(device: str):
"""Test attention backend selection with fp32."""
# Use default config (no backend specified)
vllm_config = VllmConfig()
with set_current_vllm_config(vllm_config):
if device == "cpu":
with patch("vllm.platforms.current_platform", CpuPlatform()):
backend = get_attn_backend(16, torch.float32, None)
assert backend.get_name() == "CPU_ATTN"
elif device == "cuda":
if CudaPlatform is None:
pytest.skip("CudaPlatform not available")
with patch("vllm.platforms.current_platform", CudaPlatform()):
backend = get_attn_backend(16, torch.float32, None)
assert backend.get_name() == "FLEX_ATTENTION"
elif device == "hip":
if RocmPlatform is None:
pytest.skip("RocmPlatform not available")
# ROCm backends do not support head_size=16 (minimum is 32).
# No known HuggingFace transformer model uses head_size=16.
# Revisit if a real model with this head size is identified
# and accuracy-tested.
with (
patch("vllm.platforms.current_platform", RocmPlatform()),
pytest.raises(ValueError, match="No valid attention backend"),
):
get_attn_backend(16, torch.float32, None)
def test_invalid_backend():
"""Test that invalid attention backend names raise ValueError."""
with (
pytest.raises(ValueError),
):
# Invalid backend name should raise ValueError when creating enum
AttentionConfig(backend=AttentionBackendEnum["INVALID"])
@pytest.mark.parametrize("auto_value", ["auto", "AUTO", "Auto"])
def test_auto_backend_string(auto_value: str):
"""Test that 'auto' string value triggers automatic backend selection."""
# Using "auto" should result in backend=None (automatic selection)
attention_config = AttentionConfig(backend=auto_value)
assert attention_config.backend is None
def test_auto_backend_selection_behavior():
"""Test that 'auto' backend behaves same as None (automatic selection)."""
# Create config with explicit "auto"
auto_config = AttentionConfig(backend="auto")
# Create config with None (default)
none_config = AttentionConfig(backend=None)
# Both should have backend=None
assert auto_config.backend is None
assert none_config.backend is None
# Both configs should result in the same automatic backend selection
vllm_config_auto = VllmConfig(attention_config=auto_config)
vllm_config_none = VllmConfig(attention_config=none_config)
with (
set_current_vllm_config(vllm_config_auto),
patch("vllm.platforms.current_platform", CpuPlatform()),
):
backend_auto = get_attn_backend(16, torch.float16, None)
_cached_get_attn_backend.cache_clear()
with (
set_current_vllm_config(vllm_config_none),
patch("vllm.platforms.current_platform", CpuPlatform()),
):
backend_none = get_attn_backend(16, torch.float16, None)
# Both should select the same backend
assert backend_auto.get_name() == backend_none.get_name()
@pytest.mark.parametrize(
"backend_name,flash_attn_version,should_succeed",
[
("FLASH_ATTN", 3, True), # FA3 supports per-head quant scales
("FLASH_ATTN", 2, False), # FA2 does not support per-head quant scales
("FLASHINFER", None, False), # FlashInfer does not support
("FLEX_ATTENTION", None, False), # Flex does not support
],
)
@pytest.mark.skipif(
current_platform.is_rocm(),
reason="Attention backend FA3 is not supported on ROCm. This test can't succeed.",
)
def test_per_head_quant_scales_backend_selection(
backend_name: str, flash_attn_version: Literal[2, 3, 4] | None, should_succeed: bool
):
"""Test backend selection when use_per_head_quant_scales=True."""
# Clear cache to ensure fresh backend selection
_cached_get_attn_backend.cache_clear()
attention_config = AttentionConfig(
backend=AttentionBackendEnum[backend_name],
flash_attn_version=flash_attn_version,
)
cache_config = CacheConfig(block_size=64)
vllm_config = VllmConfig(
attention_config=attention_config, cache_config=cache_config
)
if CudaPlatform is None:
pytest.skip("CudaPlatform not available")
with (
set_current_vllm_config(vllm_config),
patch("vllm.platforms.current_platform", CudaPlatform()),
):
if backend_name == "FLASH_ATTN" and flash_attn_version == 3:
if not torch.cuda.is_available():
pytest.skip("FA3 requires CUDA")
capability = torch.cuda.get_device_capability()
if capability[0] != 9:
pytest.skip("FA3 is only supported on Hopper (SM 9.x) GPUs")
if should_succeed:
backend = get_attn_backend(
head_size=128,
dtype=torch.float16,
kv_cache_dtype="fp8",
use_per_head_quant_scales=True,
)
assert backend.get_name() == backend_name
else:
with pytest.raises(ValueError) as exc_info:
get_attn_backend(
head_size=128,
dtype=torch.float16,
kv_cache_dtype="fp8",
use_per_head_quant_scales=True,
)
assert backend_name in str(exc_info.value)
@pytest.mark.skipif(
CudaPlatform is None, reason="CUDA platform is required for this test"
)
@pytest.mark.parametrize("head_size", [64, 80, 96, 128, 192, 256])
def test_int4_per_token_head_head_size_selection(head_size: int):
"""INT4 per-token-head KV cache is selectable at any even head size.
The Hadamard rotation the INT4 write path applies is now defined for
non-power-of-two rows as well (it runs block-diagonally, see
``fast_hadamard_transform``), so a model such as Phi-3 (head_size=96) must
keep selecting TRITON_ATTN instead of being rejected.
"""
vllm_config = VllmConfig(cache_config=CacheConfig(block_size=64))
with (
set_current_vllm_config(vllm_config),
patch("vllm.platforms.current_platform", CudaPlatform()),
):
backend = get_attn_backend(
head_size=head_size,
dtype=torch.float16,
kv_cache_dtype="int4_per_token_head",
)
assert backend.get_name() == "TRITON_ATTN"
@pytest.mark.parametrize(
"backend_name,use_non_causal,should_succeed",
[
("FLASH_ATTN", True, True), # FlashAttn supports non-causal
("FLASH_ATTN", False, True), # FlashAttn also works with causal
]
+ (
[
("FLASHINFER", True, True), # FlashInfer supports non-causal
("FLASHINFER", False, True), # FlashInfer works with causal
]
if CudaPlatform is not None
else []
),
)
def test_non_causal_backend_selection(
backend_name: str, use_non_causal: bool, should_succeed: bool
):
"""Test that use_non_causal on AttentionConfig controls backend filtering.
DFlashProposer sets use_non_causal=True on the draft model's
AttentionConfig so only non-causal-capable backends are selected.
The target model keeps use_non_causal=False (default) and can use
any backend.
"""
_cached_get_attn_backend.cache_clear()
attention_config = AttentionConfig(
backend=AttentionBackendEnum[backend_name],
use_non_causal=use_non_causal,
)
cache_config = CacheConfig(block_size=16)
vllm_config = VllmConfig(
attention_config=attention_config, cache_config=cache_config
)
platform = CudaPlatform or RocmPlatform
if platform is None:
pytest.skip("CudaPlatform and RocmPlatform are not available")
with (
set_current_vllm_config(vllm_config),
patch("vllm.platforms.current_platform", platform()),
):
if should_succeed:
backend = get_attn_backend(
head_size=128,
dtype=torch.float16,
kv_cache_dtype=None,
)
assert backend.get_name() == backend_name
else:
with pytest.raises(ValueError) as exc_info:
get_attn_backend(
head_size=128,
dtype=torch.float16,
kv_cache_dtype=None,
)
assert "non-causal" in str(exc_info.value).lower()
def test_non_causal_autoselect_backend():
"""Test that when backend=None with use_non_causal=True, auto-selection
picks a compatible backend.
This simulates the DFlash scenario where the user doesn't specify
--attention-backend or --speculative-config.attention_backend.
The drafter inherits backend=None and auto-selects a backend that
supports non-causal attention.
"""
_cached_get_attn_backend.cache_clear()
attention_config = AttentionConfig(
backend=None,
use_non_causal=True,
)
cache_config = CacheConfig(block_size=16)
vllm_config = VllmConfig(
attention_config=attention_config, cache_config=cache_config
)
if CudaPlatform is None:
pytest.skip("CudaPlatform not available")
with (
set_current_vllm_config(vllm_config),
patch("vllm.platforms.current_platform", CudaPlatform()),
):
backend = get_attn_backend(
head_size=128,
dtype=torch.float16,
kv_cache_dtype=None,
)
assert backend.supports_non_causal()
@pytest.mark.parametrize(
"kv_cache_dtype",
[
"fp8_e5m2",
"fp8_ds_mla",
"fp8_inc",
"nvfp4",
"nvfp4_4over6",
"fp8_per_token_head",
"int8_per_token_head",
],
)
def test_flash_attn_rejects_unhandled_kv_cache_dtypes(kv_cache_dtype: CacheDType):
"""FlashAttentionBackend must not claim support for kv_cache dtypes
that it cannot handle."""
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
assert not FlashAttentionBackend.supports_kv_cache_dtype(kv_cache_dtype)
@pytest.mark.parametrize("kv_cache_dtype", ["fp8", "fp8_e4m3"])
def test_flash_attn_accepts_handled_fp8_variants(
kv_cache_dtype: CacheDType, monkeypatch: pytest.MonkeyPatch
):
"""FlashAttentionBackend must accept the two fp8 dtypes it can actually
handle: 'fp8' (alias for fp8_e4m3fn) and 'fp8_e4m3'."""
import vllm.v1.attention.backends.fa_utils as fa_utils_mod
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
# The fp8 decision is made in fa_utils, using its own current_platform
# binding, so patch is_xpu there (not on flash_attn's) to stay robust to
# import order across earlier tests that patch vllm.platforms.current_platform.
monkeypatch.setattr(fa_utils_mod.current_platform, "is_xpu", lambda: True)
assert FlashAttentionBackend.supports_kv_cache_dtype(kv_cache_dtype)
blackwell_only = pytest.mark.skipif(
not current_platform.is_cuda(), reason="FA4 is CUDA-only"
)
@contextmanager
def _blackwell(vllm_config=None):
platform = MagicMock()
platform.is_xpu.return_value = False
platform.is_rocm.return_value = False
platform.get_device_capability.return_value = DeviceCapability(10, 0)
with (
patch("vllm.v1.attention.backends.fa_utils.current_platform", platform),
patch(
"vllm.vllm_flash_attn.flash_attn_interface.is_fa_version_supported",
return_value=True,
),
patch("vllm.config.get_current_vllm_config_or_none", return_value=vllm_config),
patch(
"vllm.v1.attention.backends.flash_attn.get_current_vllm_config_or_none",
return_value=vllm_config,
),
):
yield
def _hd256_config(
*,
is_mm_prefix_lm=False,
rswa_window=None,
dcp_size=1,
softcap=None,
head_size=256,
cache_dtype="auto",
):
vllm_config = MagicMock()
vllm_config.attention_config.flash_attn_version = None
vllm_config.model_config.is_mm_prefix_lm = is_mm_prefix_lm
vllm_config.model_config.rswa_window = rswa_window
vllm_config.model_config.hf_text_config.attn_logit_softcapping = softcap
vllm_config.model_config.get_head_size.return_value = head_size
vllm_config.cache_config.cache_dtype = cache_dtype
vllm_config.parallel_config.decode_context_parallel_size = dcp_size
return vllm_config
@blackwell_only
@pytest.mark.parametrize(
"kwargs,config_kwargs,expected",
[
({}, {}, 4),
({"supports_fa4_hd256": False}, {}, 2),
({"has_sinks": True}, {}, 2),
({"requires_softcap": True}, {}, 2),
# Larger block sizes normalize to 128.
({"kv_cache_block_size": 16}, {}, 2),
({"kv_cache_block_size": 64}, {}, 2),
({"kv_cache_block_size": 256}, {}, 4),
({}, {"is_mm_prefix_lm": True}, 2),
({}, {"rswa_window": 512}, 2),
({}, {"dcp_size": 2}, 2),
({}, {"softcap": 50.0}, 2),
({}, {"cache_dtype": "fp8"}, 2),
({"head_size": 128, "kv_cache_block_size": 16}, {}, 4),
({"head_size": 192, "head_size_v": 128, "kv_cache_block_size": 16}, {}, 4),
({"head_size": 256, "head_size_v": 128, "kv_cache_block_size": 16}, {}, 2),
({"head_size": 256, "head_size_v": 64}, {}, 2),
],
)
def test_fa4_hd256_fallback_matrix(kwargs, config_kwargs, expected):
from vllm.v1.attention.backends.fa_utils import get_flash_attn_version
kwargs = {
"head_size": 256,
"kv_cache_block_size": 128,
"supports_fa4_hd256": True,
**kwargs,
}
with _blackwell(_hd256_config(head_size=kwargs["head_size"], **config_kwargs)):
assert get_flash_attn_version(**kwargs) == expected
@blackwell_only
@pytest.mark.parametrize(
"backend_name,config_kwargs,expected",
[
("FLASH_ATTN", {}, 128),
("FLASH_ATTN", {"softcap": 50.0}, 16),
("FLASH_ATTN", {"head_size": 128}, 16),
("FLASH_ATTN", {"cache_dtype": "fp8"}, 16),
("FLASH_ATTN_DIFFKV", {}, 16),
],
)
def test_fa4_hd256_block_size_advertisement(
backend_name: str, config_kwargs: dict, expected: int
):
from vllm.v1.attention.backend import MultipleOf
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
from vllm.v1.attention.backends.flash_attn_diffkv import (
FlashAttentionDiffKVBackend,
)
backend = {
"FLASH_ATTN": FlashAttentionBackend,
"FLASH_ATTN_DIFFKV": FlashAttentionDiffKVBackend,
}[backend_name]
vllm_config = _hd256_config(**config_kwargs)
with _blackwell(vllm_config):
(size,) = backend.get_supported_kernel_block_sizes()
preferred = backend.get_preferred_block_size(16)
assert preferred == expected
if expected == 128:
assert size == 128
else:
assert isinstance(size, MultipleOf) and size.base == expected
@blackwell_only
def test_fa4_hd256_mm_prefix_deselects_flash_attn():
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
with _blackwell(_hd256_config(is_mm_prefix_lm=True)):
assert (
FlashAttentionBackend.supports_combination(
head_size=256,
dtype=torch.bfloat16,
kv_cache_dtype=None,
block_size=128,
use_mla=False,
has_sink=False,
use_sparse=False,
use_mm_prefix=True,
device_capability=DeviceCapability(10, 0),
)
is not None
)
@pytest.mark.skipif(
not current_platform.is_cuda()
or not current_platform.is_device_capability_family(100),
reason="requires a Blackwell GPU",
)
@pytest.mark.parametrize(
"attn_type,sliding_window,expected",
[
(AttentionType.DECODER, None, 4),
(AttentionType.DECODER, 512, 4),
# Local encoder attention lacks the required seqused tensors.
(AttentionType.ENCODER_ONLY, None, 4),
(AttentionType.ENCODER_ONLY, 512, 2),
],
)
def test_fa4_hd256_impl_selection(attn_type, sliding_window, expected):
from vllm.v1.attention.backends.flash_attn import FlashAttentionImpl
# Implementations are constructed before the platform updates the block size.
with set_current_vllm_config(VllmConfig()):
impl = FlashAttentionImpl(
num_heads=8,
head_size=256,
scale=0.0625,
num_kv_heads=8,
alibi_slopes=None,
sliding_window=sliding_window,
kv_cache_dtype="auto",
attn_type=attn_type,
)
assert impl.vllm_flash_attn_version == expected
assert impl.fa4_hd256 == (expected == 4)
@pytest.fixture
def blackwell_selection():
with (
patch.object(
type(current_platform),
"get_device_capability",
return_value=DeviceCapability(10, 0),
),
patch(
"vllm.v1.attention.backends.fa_utils.is_fa_version_supported",
return_value=True,
),
):
yield
@pytest.fixture
def hopper_selection():
with (
patch.object(
type(current_platform),
"get_device_capability",
return_value=DeviceCapability(9, 0),
),
patch(
"vllm.v1.attention.backends.fa_utils.is_fa_version_supported",
return_value=True,
),
):
yield
@blackwell_only
@pytest.mark.parametrize("use_mm_prefix", [False, True])
@pytest.mark.parametrize("flash_attn_version", [None, 3, 4])
def test_hopper_mm_prefix_selects_triton_flash_attn(
use_mm_prefix, flash_attn_version, hopper_selection
):
"""Hopper selects the composite when its causal route resolves FA3 or FA4."""
from vllm.engine.arg_utils import EngineArgs
config = EngineArgs(
model="google/gemma-4-31B-it",
dtype="bfloat16",
attention_config=AttentionConfig(flash_attn_version=flash_attn_version),
).create_engine_config()
with set_current_vllm_config(config):
backend = get_attn_backend(
256, torch.bfloat16, None, use_mm_prefix=use_mm_prefix
)
expected = "TRITON_FLASH_ATTN" if use_mm_prefix else "FLASH_ATTN"
assert backend.get_name() == expected
@blackwell_only
@pytest.mark.parametrize("use_mm_prefix", [False, True])
@pytest.mark.parametrize("kv_cache_dtype", [None, "fp8_e4m3"])
def test_mm_prefix_selects_composite_without_changing_causal_default(
use_mm_prefix, kv_cache_dtype, blackwell_selection
):
"""The image-mask requirement must reach CUDA's automatic backend priority."""
from vllm.engine.arg_utils import EngineArgs
config = EngineArgs(
model="google/gemma-4-31B-it", dtype="bfloat16"
).create_engine_config()
with set_current_vllm_config(config):
backend = get_attn_backend(
256, torch.bfloat16, kv_cache_dtype, use_mm_prefix=use_mm_prefix
)
expected = "TRITON_FLASHINFER" if use_mm_prefix else "FLASHINFER"
assert backend.get_name() == expected
@blackwell_only
@pytest.mark.parametrize(
"feature_kwargs,reason",
[
({"use_rswa": True}, "R-SWA not supported"),
({"kv_cache_dtype": "int8_per_token_head"}, "kv_cache_dtype not supported"),
({"block_size": 16}, "block_size not supported"),
({"use_dcp": True}, "DCP not supported"),
({"use_pcp": True}, "PCP not supported"),
({"use_adaptive_verification": True}, "device-cpu query lens mismatch"),
],
)
def test_composite_rejects_features_not_shared_by_both_routes(feature_kwargs, reason):
"""Automatic selection must not admit features only one child can execute."""
from vllm.engine.arg_utils import EngineArgs
from vllm.v1.attention.backends.triton_flashinfer import TritonFlashInferBackend
config = EngineArgs(
model="google/gemma-4-31B-it", dtype="bfloat16"
).create_engine_config()
args = dict(
head_size=256,
dtype=torch.bfloat16,
kv_cache_dtype="auto",
block_size=128,
use_mla=False,
has_sink=False,
use_sparse=False,
use_mm_prefix=True,
use_per_head_quant_scales=False,
device_capability=DeviceCapability(10, 0),
attn_type=AttentionType.DECODER,
)
args.update(feature_kwargs)
with set_current_vllm_config(config):
reasons = TritonFlashInferBackend.validate_configuration(**args)
assert any(reason in message for message in reasons), reasons
@blackwell_only
def test_rswa_selection_does_not_reuse_causal_result(blackwell_selection):
"""An R-SWA requirement must invalidate a previously cached causal selection."""
from vllm.engine.arg_utils import EngineArgs
config = EngineArgs(
model="google/gemma-4-31B-it",
dtype="bfloat16",
attention_config=AttentionConfig(
backend=AttentionBackendEnum.TRITON_FLASHINFER
),
).create_engine_config()
with set_current_vllm_config(config):
assert (
get_attn_backend(256, torch.bfloat16, None).get_name()
== "TRITON_FLASHINFER"
)
config.model_config.model_arch_config.rswa_window = 128
with pytest.raises(ValueError, match="R-SWA"):
get_attn_backend(256, torch.bfloat16, None)
config.attention_config.backend = AttentionBackendEnum.TRITON_ATTN
assert get_attn_backend(256, torch.bfloat16, None).get_name() == "TRITON_ATTN"
File diff suppressed because it is too large Load Diff
@@ -1,410 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import random
import pytest
import torch
from vllm.model_executor.layers.sparse_attn_indexer import as_fp32_logits
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import (
_ceil_to_ue8m0,
calc_diff,
fp8_fp4_mqa_logits,
fp8_fp4_paged_mqa_logits,
get_num_sms,
get_paged_mqa_logits_metadata,
mqa_logits_mx_inputs,
mqa_logits_use_mx_scales,
native_next_n_supported,
)
from vllm.utils.import_utils import has_deep_gemm
from vllm.utils.math_utils import cdiv
def encode_indexer_scales(sf: torch.Tensor) -> torch.Tensor:
"""Per-token power-of-two fp32 scales as the indexer K cache stores them:
four equal packed UE8M0 bytes under MX scales, else the fp32 itself."""
if not mqa_logits_use_mx_scales():
return sf
ue8m0 = (sf.view(torch.int32) >> 23).to(torch.uint8)
return ue8m0[..., None].repeat_interleave(4, dim=-1).view(torch.int32)[..., 0]
def kv_cache_cast_to_fp8(x: torch.Tensor) -> torch.Tensor:
# x: (num_blocks, block_size, 1, head_dim)
num_blocks, block_size, num_heads, head_dim = x.shape
assert num_heads == 1
x_amax = x.abs().float().amax(dim=3, keepdim=True).clamp(1e-4)
sf = _ceil_to_ue8m0(x_amax / 448.0)
x_scaled = (x * (1.0 / sf)).to(torch.float8_e4m3fn)
x_fp8 = torch.empty(
(num_blocks, block_size * (head_dim + 4)),
device=x.device,
dtype=torch.uint8,
)
x_fp8[:, : block_size * head_dim] = x_scaled.view(
num_blocks, block_size * head_dim
).view(dtype=torch.uint8)
x_fp8[:, block_size * head_dim :] = encode_indexer_scales(
sf.view(num_blocks, block_size)
).view(dtype=torch.uint8)
return x_fp8.view(num_blocks, block_size, num_heads, head_dim + 4)
def per_custom_dims_cast_to_fp8(
x: torch.Tensor, dims: tuple, use_ue8m0: bool
) -> tuple[torch.Tensor, torch.Tensor]:
excluded_dims = tuple([i for i in range(x.dim()) if i not in set(dims)])
x_amax = x.abs().float().amax(dim=excluded_dims, keepdim=True).clamp(1e-4)
sf = x_amax / 448.0
sf = _ceil_to_ue8m0(sf) if use_ue8m0 else sf
x_scaled = (x * (1.0 / sf)).to(torch.float8_e4m3fn)
return x_scaled, sf.squeeze()
def _generate_cp_test_data(seq_len: int, seq_len_kv: int):
assert seq_len_kv % seq_len == 0 and seq_len % 2 == 0
chunk_size = seq_len // 2
cp_size = seq_len_kv // seq_len
cp_id = cp_size // 3
ks = torch.zeros(seq_len, dtype=torch.int, device="cuda")
ke = torch.zeros(seq_len, dtype=torch.int, device="cuda")
for i in range(chunk_size):
ke[i] = cp_id * chunk_size + i
ke[i + chunk_size] = (cp_size * 2 - 1 - cp_id) * chunk_size + i
return ks, ke
def _ref_fp8_mqa_logits(
q: torch.Tensor,
kv: torch.Tensor,
weights: torch.Tensor,
cu_seqlen_ks: torch.Tensor,
cu_seqlen_ke: torch.Tensor,
):
seq_len_kv = kv.shape[0]
k = kv
q = q.float()
k = k.float()
mask_lo = (
torch.arange(0, seq_len_kv, device="cuda")[None, :] >= cu_seqlen_ks[:, None]
)
mask_hi = (
torch.arange(0, seq_len_kv, device="cuda")[None, :] < cu_seqlen_ke[:, None]
)
mask = mask_lo & mask_hi
score = torch.einsum("mhd,nd->hmn", q, k)
logits = (score.relu() * weights.unsqueeze(-1).transpose(0, 1)).sum(dim=0)
logits = logits.masked_fill(~mask, float("-inf"))
return logits
def _packed_requests_test_data(seq_len: int, seq_len_kv: int):
# Two causal prefill requests packed into one KV buffer, so the second
# request's rows start at a nonzero K position.
half, half_kv = seq_len // 2, seq_len_kv // 2
rows = torch.arange(seq_len, dtype=torch.int, device="cuda")
ks = torch.where(rows < half, 0, half_kv).to(torch.int)
ke = ks + half_kv - half + 1 + rows % half
return ks, ke
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
@pytest.mark.skipif(not has_deep_gemm(), reason="DeepGEMM not available")
@pytest.mark.skipif(
not current_platform.has_device_capability(90), reason="SM90 and SM100 only"
)
@pytest.mark.parametrize("layout", ["cp", "no_cp", "packed"])
def test_deepgemm_fp8_mqa_logits(layout: str):
"""FP8 indexer prefill logits, compressed to each row's valid K span."""
torch.manual_seed(0)
random.seed(0)
num_heads, head_dim = 32, 128
seq_len, seq_len_kv = 512, 1024
q = torch.randn(seq_len, num_heads, head_dim, device="cuda", dtype=torch.bfloat16)
kv = torch.randn(seq_len_kv, head_dim, device="cuda", dtype=torch.bfloat16)
weights = torch.randn(seq_len, num_heads, device="cuda", dtype=torch.float32)
if layout == "cp":
ks, ke = _generate_cp_test_data(seq_len, seq_len_kv)
elif layout == "no_cp":
ks = torch.zeros(seq_len, dtype=torch.int, device="cuda")
ke = torch.arange(seq_len, dtype=torch.int, device="cuda") + (
seq_len_kv - seq_len
)
else:
ks, ke = _packed_requests_test_data(seq_len, seq_len_kv)
q_fp8 = q.to(torch.float8_e4m3fn)
kv_fp8, kv_sf = per_custom_dims_cast_to_fp8(kv, (0,), True)
q_scale, kernel_weights = mqa_logits_mx_inputs(q_fp8, None, weights)
max_seqlen_k = int((ke - ks).max())
logits = fp8_fp4_mqa_logits(
(q_fp8, q_scale),
(kv_fp8, encode_indexer_scales(kv_sf)),
kernel_weights,
ks,
ke,
max_seqlen_k,
)
assert logits.shape == (seq_len, max_seqlen_k)
logits = as_fp32_logits(logits, ke - ks)
ref_logits = _ref_fp8_mqa_logits(
q=q,
kv=kv,
weights=kernel_weights.float(),
cu_seqlen_ks=ks,
cu_seqlen_ke=ke,
)
# Compressed row i holds K positions [ks[i], ke[i]) from column zero.
cols = ks[:, None] + torch.arange(max_seqlen_k, device="cuda")[None, :]
valid = cols < ke[:, None]
ref_logits = ref_logits.gather(1, cols.clamp(max=seq_len_kv - 1).long())
ref_logits = ref_logits.masked_fill(~valid, 0)
logits = logits.masked_fill(~valid, 0)
diff = calc_diff(logits, ref_logits)
assert diff < 1e-3, f"{diff=}"
def _ref_fp8_fp4_paged_mqa_logits(
q: torch.Tensor,
kv_cache: torch.Tensor,
weights: torch.Tensor,
context_lens: torch.Tensor,
block_tables: torch.Tensor,
max_model_len: int,
):
batch_size, next_n, _, _ = q.size()
_, block_size, _, _ = kv_cache.size()
logits = torch.full(
[batch_size * next_n, max_model_len],
float("-inf"),
device=q.device,
dtype=torch.float32,
)
context_lens_list = context_lens.tolist()
for i in range(batch_size):
context_len = context_lens_list[i]
q_offsets = torch.arange(context_len - next_n, context_len, device="cuda")
weight_slice = (
weights[i * next_n : (i + 1) * next_n, :].transpose(0, 1).contiguous()
)
for block_rk in range(cdiv(context_len, block_size)):
block_idx = block_tables[i][block_rk]
qx, kx = q[i], kv_cache[block_idx]
k_offsets = torch.arange(
block_rk * block_size,
(block_rk + 1) * block_size,
device="cuda",
)
mask = (k_offsets[None, :] < context_len) & (
k_offsets[None, :] <= q_offsets[:, None]
)
s = torch.where(
mask[None, :, :],
(qx.transpose(0, 1) @ kx.transpose(0, 1).transpose(1, 2)).to(
logits.dtype
),
float("-inf"),
)
s = torch.relu(s) * weight_slice[..., None]
s = s.sum(dim=0)
logits[
i * next_n : (i + 1) * next_n,
block_rk * block_size : (block_rk + 1) * block_size,
] = torch.where(k_offsets[None, :] <= q_offsets[:, None], s, float("-inf"))
return logits
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
@pytest.mark.skipif(not has_deep_gemm(), reason="DeepGEMM not available")
@pytest.mark.skipif(
not current_platform.has_device_capability(90), reason="SM90 and SM100 only"
)
# next_n = 1 + num_speculative_tokens, so next_n=4 is MTP=3 (issue #35878).
@pytest.mark.parametrize("batch_size,next_n", [(4, 1), (2, 2), (2, 4)])
def test_deepgemm_fp8_fp4_paged_mqa_logits(batch_size: int, next_n: int):
if not native_next_n_supported(next_n):
pytest.skip(f"next_n={next_n} has no native kernel on this architecture")
torch.manual_seed(0)
random.seed(0)
max_model_len = 4096
for heads, index_dim in [(32, 128)]:
for avg_kv in (2048,):
num_blocks, blocksize = max_model_len * 2, 64
q = torch.randn(
(batch_size, next_n, heads, index_dim),
device="cuda",
dtype=torch.bfloat16,
)
kv_cache = torch.randn(
(num_blocks, blocksize, 1, index_dim),
device="cuda",
dtype=torch.bfloat16,
)
weights = torch.randn(
(batch_size * next_n, heads),
device="cuda",
dtype=torch.float32,
)
context_lens = (
torch.randint(int(0.8 * avg_kv), int(1.2 * avg_kv), (batch_size,))
.cuda()
.to(torch.int32)
)
max_block_len = (
(context_lens.max().item() + blocksize - 1) // blocksize * blocksize
)
block_tables = torch.zeros(
(batch_size, max_block_len),
device="cuda",
dtype=torch.int32,
)
counter = 0
block_idx_pool = list(range(num_blocks))
random.shuffle(block_idx_pool)
for i in range(batch_size):
ctx_len = int(context_lens[i].item())
for j in range((ctx_len + blocksize - 1) // blocksize):
block_tables[i][j] = block_idx_pool[counter]
counter += 1
q_fp8 = q.to(torch.float8_e4m3fn)
kv_cache_fp8 = kv_cache_cast_to_fp8(kv_cache)
# deep_gemm paged MQA logits requires 2D context_lens of
# shape (B, next_n) (csrc/apis/attention.hpp:332-335);
# see indexer.py:607-608. For each batch/next_n token, the
# effective context length is context_lens[b] - next_n + j + 1.
next_n_arange = torch.arange(next_n, device="cuda", dtype=torch.int32)
context_lens_2d = (
context_lens.unsqueeze(-1) - next_n + 1 + next_n_arange
).contiguous()
q_scale, kernel_weights = mqa_logits_mx_inputs(q_fp8, None, weights)
kernel_q = (q_fp8, q_scale)
kernel_context_lens, kernel_block_tables = context_lens_2d, block_tables
indices = None
if mqa_logits_use_mx_scales():
# SM100 only takes varlen rows: one Q token each, tied to its
# request by `indices`.
assert q_scale is not None
kernel_q = (
q_fp8.flatten(0, 1)[:, None],
q_scale.flatten(0, 1)[:, None],
)
kernel_context_lens = context_lens_2d.view(-1, 1)
kernel_block_tables = block_tables.repeat_interleave(next_n, dim=0)
indices = torch.arange(batch_size, device="cuda", dtype=torch.int32)
indices = indices.repeat_interleave(next_n)
schedule_metadata = get_paged_mqa_logits_metadata(
kernel_context_lens, blocksize, get_num_sms(), indices=indices
)
logits = fp8_fp4_paged_mqa_logits(
kernel_q,
kv_cache_fp8,
kernel_weights,
kernel_context_lens,
kernel_block_tables,
schedule_metadata,
max_model_len,
indices=indices,
)
if indices is not None:
logits = as_fp32_logits(logits, kernel_context_lens.view(-1))
ref_logits = _ref_fp8_fp4_paged_mqa_logits(
q,
kv_cache,
kernel_weights.float(),
context_lens,
block_tables,
max_model_len,
)
positions = (
torch.arange(max_model_len, device="cuda")
.unsqueeze(0)
.expand(batch_size * next_n, -1)
)
row_indices = torch.arange(batch_size * next_n, device="cuda") // next_n
next_n_offset = torch.arange(batch_size * next_n, device="cuda") % next_n
mask = positions <= (
context_lens[row_indices] - next_n + next_n_offset
).unsqueeze(1)
logits = logits.float().masked_fill(~mask, 0)
ref_logits = ref_logits.masked_fill(~mask, 0)
diff = calc_diff(logits, ref_logits)
assert diff < 1e-3, f"{diff=}"
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
@pytest.mark.skipif(not has_deep_gemm(), reason="DeepGEMM not available")
@pytest.mark.skipif(
not current_platform.has_device_capability(90), reason="SM90 and SM100 only"
)
def test_deepgemm_paged_mqa_packed_manager_block_stride():
torch.manual_seed(0)
num_blocks, block_size, packed_rows = 4, 256, 320
num_heads, head_dim, row_bytes = 64, 128, 132
page_size, pages_per_block = 64, block_size // 64
dense_stride, packed_stride = pages_per_block, packed_rows // page_size
compact_pages = kv_cache_cast_to_fp8(
torch.randn(
num_blocks, block_size, 1, head_dim, device="cuda", dtype=torch.bfloat16
).view(num_blocks * pages_per_block, page_size, 1, head_dim)
)
packed_backing = torch.zeros(
num_blocks,
packed_rows,
row_bytes,
device="cuda",
dtype=torch.uint8,
)
packed_manager = packed_backing[:, :block_size]
packed_manager.copy_(compact_pages.squeeze(2).view(num_blocks, block_size, -1))
packed_pages = packed_backing.view(-1, page_size, row_bytes).unsqueeze(2)
offsets = torch.arange(pages_per_block, device="cuda", dtype=torch.int32)
tables = [
(stride + offsets).unsqueeze(0) for stride in (dense_stride, packed_stride)
]
q = torch.randn(1, 1, num_heads, head_dim, device="cuda", dtype=torch.bfloat16).to(
torch.float8_e4m3fn
)
q_scale, weights = mqa_logits_mx_inputs(
q, None, torch.randn(1, num_heads, device="cuda", dtype=torch.float32)
)
context_lens = torch.full((1, 1), block_size, device="cuda", dtype=torch.int32)
indices = (
torch.zeros(1, device="cuda", dtype=torch.int32)
if mqa_logits_use_mx_scales()
else None
)
schedule = get_paged_mqa_logits_metadata(
context_lens, page_size, get_num_sms(), indices=indices
)
logits = [
fp8_fp4_paged_mqa_logits(
(q, q_scale),
cache,
weights,
context_lens,
table,
schedule,
block_size,
indices=indices,
)
for cache, table in zip((compact_pages, packed_pages), tables)
]
torch.testing.assert_close(*logits, rtol=0, atol=0)
-737
View File
@@ -1,737 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
from vllm.v1.kv_cache_layout import KVCacheLayout
try:
import flashinfer
except ImportError:
if current_platform.is_rocm():
pytest.skip(
"flashinfer is not supported for vLLM on ROCm.", allow_module_level=True
)
import torch
NUM_HEADS = [(32, 8), (6, 1)]
HEAD_SIZES = [128, 256]
BLOCK_SIZES = [16, 32]
DTYPES = [torch.bfloat16]
NUM_BLOCKS = 32768 # Large enough to test overflow in index calculation.
SOFT_CAPS = [None, 30.0]
SLIDING_WINDOWS = [None, 64]
def ref_paged_attn(
query: torch.Tensor,
key_cache: torch.Tensor,
value_cache: torch.Tensor,
query_lens: list[int],
kv_lens: list[int],
block_tables: torch.Tensor,
scale: float,
sliding_window: int | None = None,
soft_cap: float | None = None,
) -> torch.Tensor:
num_seqs = len(query_lens)
block_tables = block_tables.cpu().numpy()
_, block_size, num_kv_heads, head_size = key_cache.shape
outputs: list[torch.Tensor] = []
start_idx = 0
for i in range(num_seqs):
query_len = query_lens[i]
kv_len = kv_lens[i]
q = query[start_idx : start_idx + query_len]
q *= scale
num_kv_blocks = (kv_len + block_size - 1) // block_size
block_indices = block_tables[i, :num_kv_blocks]
k = key_cache[block_indices].view(-1, num_kv_heads, head_size)
k = k[:kv_len]
v = value_cache[block_indices].view(-1, num_kv_heads, head_size)
v = v[:kv_len]
if q.shape[1] != k.shape[1]:
k = torch.repeat_interleave(k, q.shape[1] // k.shape[1], dim=1)
v = torch.repeat_interleave(v, q.shape[1] // v.shape[1], dim=1)
attn = torch.einsum("qhd,khd->hqk", q, k).float()
empty_mask = torch.ones(query_len, kv_len)
mask = torch.triu(empty_mask, diagonal=kv_len - query_len + 1).bool()
if sliding_window is not None:
sliding_window_mask = (
torch.triu(
empty_mask, diagonal=kv_len - (query_len + sliding_window) + 1
)
.bool()
.logical_not()
)
mask |= sliding_window_mask
if soft_cap is not None:
attn = soft_cap * torch.tanh(attn / soft_cap)
attn.masked_fill_(mask, float("-inf"))
attn = torch.softmax(attn, dim=-1).to(v.dtype)
out = torch.einsum("hqk,khd->qhd", attn, v)
outputs.append(out)
start_idx += query_len
return torch.cat(outputs, dim=0)
def _make_paged_kv_metadata(
kv_lens: list[int],
block_size: int,
num_blocks: int,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""Build paged-KV metadata tensors for fast_plan_decode tests.
Returns:
kv_indptr – CPU int32, shape [num_seqs + 1]
kv_indices – CUDA int32, shape [total_blocks]
kv_last_page_lens – CPU int32, shape [num_seqs]
block_tables – CUDA int32, shape [num_seqs, max_blocks_per_seq]
"""
num_seqs = len(kv_lens)
max_blocks = (max(kv_lens) + block_size - 1) // block_size
block_tables = torch.randint(
0, num_blocks, (num_seqs, max_blocks), dtype=torch.int32, device="cuda"
)
indptr_list = [0]
indices_list: list[int] = []
last_lens_list: list[int] = []
for i, seq_len in enumerate(kv_lens):
n = (seq_len + block_size - 1) // block_size
indices_list.extend(block_tables[i, :n].cpu().tolist())
indptr_list.append(indptr_list[-1] + n)
last_lens_list.append(seq_len % block_size or block_size)
return (
torch.tensor(indptr_list, dtype=torch.int32, device="cpu"),
torch.tensor(indices_list, dtype=torch.int32, device="cuda"),
torch.tensor(last_lens_list, dtype=torch.int32, device="cpu"),
block_tables,
)
def _make_cg_decode_wrapper(
num_seqs: int,
kv_indices_buffer: torch.Tensor,
workspace_buffer: torch.Tensor,
use_tensor_cores: bool = True,
) -> "flashinfer.BatchDecodeWithPagedKVCacheWrapper":
"""Create a cudagraph-enabled BatchDecodeWithPagedKVCacheWrapper.
*kv_indices_buffer* is shared with the caller so that fast_plan_decode
can avoid the device-to-device index copy on subsequent (cudagraph) calls.
"""
return flashinfer.BatchDecodeWithPagedKVCacheWrapper(
workspace_buffer,
"NHD",
use_cuda_graph=True,
paged_kv_indptr_buffer=torch.zeros(
num_seqs + 1, dtype=torch.int32, device="cuda"
),
paged_kv_indices_buffer=kv_indices_buffer,
paged_kv_last_page_len_buffer=torch.zeros(
num_seqs, dtype=torch.int32, device="cuda"
),
use_tensor_cores=use_tensor_cores,
)
def test_fast_decode_plan_importable() -> None:
"""fast_decode_plan must be importable from flashinfer.decode.
This is a forward-compatibility smoke test: if FlashInfer reorganises its
public API the import will fail before any other test does.
"""
from flashinfer.decode import fast_decode_plan # noqa: F401
assert callable(fast_decode_plan)
@pytest.mark.parametrize("dtype", DTYPES)
@torch.inference_mode
def test_fast_plan_decode_warmup_uses_full_plan(dtype: torch.dtype) -> None:
"""On the first call fast_plan_decode must route through self.plan() and
flip vllm_first_call to False on the wrapper object."""
from unittest.mock import patch
from vllm.v1.attention.backends.flashinfer import fast_plan_decode
torch.set_default_device("cuda")
set_random_seed(0)
kv_lens = [128, 64]
block_size = 16
num_seqs = len(kv_lens)
num_query_heads, num_kv_heads = 8, 2
head_size = 128
kv_indptr, kv_indices, kv_last_page_lens, _ = _make_paged_kv_metadata(
kv_lens, block_size, NUM_BLOCKS
)
workspace = torch.empty(128 * 1024 * 1024, dtype=torch.int8)
wrapper = _make_cg_decode_wrapper(num_seqs, kv_indices.clone(), workspace)
assert getattr(wrapper, "vllm_first_call", True) is True
with patch.object(wrapper, "plan", wraps=wrapper.plan) as mock_plan:
fast_plan_decode(
wrapper,
indptr_cpu=kv_indptr,
indices=kv_indices,
last_page_len_cpu=kv_last_page_lens,
seq_lens_cpu=torch.tensor(kv_lens, dtype=torch.int32, device="cpu"),
num_qo_heads=num_query_heads,
num_kv_heads=num_kv_heads,
head_dim=head_size,
page_size=block_size,
q_data_type=dtype,
kv_data_type=dtype,
)
mock_plan.assert_called_once()
assert wrapper.vllm_first_call is False, (
"vllm_first_call should be False after the first fast_plan_decode call"
)
@pytest.mark.parametrize("kv_lens", [[1328, 18, 463], [1, 54, 293, 70]])
@pytest.mark.parametrize("num_heads", NUM_HEADS)
@pytest.mark.parametrize("head_size", HEAD_SIZES)
@pytest.mark.parametrize("block_size", BLOCK_SIZES)
@pytest.mark.parametrize("dtype", DTYPES)
@torch.inference_mode
def test_fast_plan_decode_matches_full_plan(
kv_lens: list[int],
num_heads: tuple[int, int],
head_size: int,
block_size: int,
dtype: torch.dtype,
) -> None:
"""fast_plan_decode's cudagraph path (delegating to FlashInfer's
fast_decode_plan) must produce attention output numerically identical to
a standard plan() call.
Both the warmup call (self.plan) and the subsequent fast call
(fast_decode_plan) are verified against the same reference.
"""
from vllm.v1.attention.backends.flashinfer import fast_plan_decode
torch.set_default_device("cuda")
set_random_seed(0)
num_seqs = len(kv_lens)
num_query_heads, num_kv_heads = num_heads
query = torch.randn(num_seqs, num_query_heads, head_size, dtype=dtype)
key_value_cache = torch.randn(
NUM_BLOCKS, 2, block_size, num_kv_heads, head_size, dtype=dtype
)
kv_indptr, kv_indices, kv_last_page_lens, _ = _make_paged_kv_metadata(
kv_lens, block_size, NUM_BLOCKS
)
# Reference output via the standard plan()
workspace_ref = torch.empty(128 * 1024 * 1024, dtype=torch.int8)
ref_wrapper = flashinfer.BatchDecodeWithPagedKVCacheWrapper(
workspace_ref, "NHD", use_tensor_cores=True
)
ref_wrapper.plan(
kv_indptr,
kv_indices,
kv_last_page_lens,
num_query_heads,
num_kv_heads,
head_size,
block_size,
"NONE",
q_data_type=dtype,
kv_data_type=dtype,
)
ref_output = ref_wrapper.run(query, key_value_cache)
# CUDAGraph wrapper exercised through fast_plan_decode
kv_indices_buf = kv_indices.clone()
workspace_cg = torch.empty(128 * 1024 * 1024, dtype=torch.int8)
cg_wrapper = _make_cg_decode_wrapper(num_seqs, kv_indices_buf, workspace_cg)
plan_kwargs: dict = dict(
indptr_cpu=kv_indptr,
indices=kv_indices_buf,
last_page_len_cpu=kv_last_page_lens,
seq_lens_cpu=torch.tensor(kv_lens, dtype=torch.int32, device="cpu"),
num_qo_heads=num_query_heads,
num_kv_heads=num_kv_heads,
head_dim=head_size,
page_size=block_size,
q_data_type=dtype,
kv_data_type=dtype,
)
# First call – warmup path (routes through self.plan)
fast_plan_decode(cg_wrapper, **plan_kwargs)
warmup_output = cg_wrapper.run(query, key_value_cache)
torch.testing.assert_close(warmup_output, ref_output, atol=1e-2, rtol=1e-2)
# Second call – fast path (routes through fast_decode_plan from FlashInfer)
fast_plan_decode(cg_wrapper, **plan_kwargs)
fast_output = cg_wrapper.run(query, key_value_cache)
torch.testing.assert_close(fast_output, ref_output, atol=1e-2, rtol=1e-2)
@pytest.mark.parametrize("kv_lens", [[1328, 18, 463], [1, 54, 293, 70]])
@pytest.mark.parametrize("num_heads", NUM_HEADS)
@pytest.mark.parametrize("head_size", HEAD_SIZES)
@pytest.mark.parametrize("block_size", BLOCK_SIZES)
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("soft_cap", SOFT_CAPS)
@pytest.mark.parametrize("sliding_window", SLIDING_WINDOWS)
@torch.inference_mode
def test_flashinfer_decode_with_paged_kv(
kv_lens: list[int],
num_heads: tuple[int, int],
head_size: int,
dtype: torch.dtype,
block_size: int,
soft_cap: float | None,
sliding_window: int | None,
) -> None:
torch.set_default_device("cuda")
set_random_seed(0)
num_seqs = len(kv_lens)
num_query_heads = num_heads[0]
num_kv_heads = num_heads[1]
assert num_query_heads % num_kv_heads == 0
max_kv_len = max(kv_lens)
scale = head_size**-0.5
query = torch.randn(num_seqs, num_query_heads, head_size, dtype=dtype)
key_value_cache = torch.randn(
NUM_BLOCKS, 2, block_size, num_kv_heads, head_size, dtype=dtype
)
key_cache = key_value_cache[:, 0, :, :, :].squeeze(1)
value_cache = key_value_cache[:, 1, :, :, :].squeeze(1)
max_num_blocks_per_seq = (max_kv_len + block_size - 1) // block_size
block_tables = torch.randint(
0, NUM_BLOCKS, (num_seqs, max_num_blocks_per_seq), dtype=torch.int32
)
kv_indptr = [0]
kv_indices = []
kv_last_page_lens = []
for i in range(num_seqs):
seq_len = kv_lens[i]
assert seq_len > 0
num_blocks = (seq_len + block_size - 1) // block_size
kv_indices.extend(block_tables[i, :num_blocks])
kv_indptr.append(kv_indptr[-1] + num_blocks)
kv_last_page_len = seq_len % block_size
if kv_last_page_len == 0:
kv_last_page_len = block_size
kv_last_page_lens.append(kv_last_page_len)
kv_indptr = torch.tensor(kv_indptr, dtype=torch.int32)
kv_indices = torch.tensor(kv_indices, dtype=torch.int32)
kv_last_page_lens = torch.tensor(kv_last_page_lens, dtype=torch.int32)
workspace_buffer = torch.empty(128 * 1024 * 1024, dtype=torch.int8)
wrapper = flashinfer.BatchDecodeWithPagedKVCacheWrapper(
workspace_buffer, "NHD", use_tensor_cores=True
)
wrapper.plan(
kv_indptr,
kv_indices,
kv_last_page_lens,
num_query_heads,
num_kv_heads,
head_size,
block_size,
"NONE",
window_left=sliding_window - 1 if sliding_window is not None else -1,
q_data_type=dtype,
kv_data_type=dtype,
logits_soft_cap=soft_cap,
)
output = wrapper.run(query, key_value_cache)
ref_output = ref_paged_attn(
query=query,
key_cache=key_cache,
value_cache=value_cache,
query_lens=[1] * num_seqs,
kv_lens=kv_lens,
block_tables=block_tables,
scale=scale,
soft_cap=soft_cap,
sliding_window=sliding_window,
)
(
torch.testing.assert_close(output, ref_output, atol=1e-2, rtol=1e-2),
f"{torch.max(torch.abs(output - ref_output))}",
)
@pytest.mark.parametrize("seq_lens", [[(1, 1328), (5, 18), (129, 463)]])
@pytest.mark.parametrize("num_heads", NUM_HEADS)
@pytest.mark.parametrize("head_size", HEAD_SIZES)
@pytest.mark.parametrize("block_size", BLOCK_SIZES)
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("soft_cap", SOFT_CAPS)
@pytest.mark.parametrize("sliding_window", SLIDING_WINDOWS)
@torch.inference_mode
def test_flashinfer_prefill_with_paged_kv(
seq_lens: list[tuple[int, int]],
num_heads: tuple[int, int],
head_size: int,
dtype: torch.dtype,
block_size: int,
soft_cap: float | None,
sliding_window: int | None,
) -> None:
torch.set_default_device("cuda")
set_random_seed(0)
num_seqs = len(seq_lens)
query_lens = [x[0] for x in seq_lens]
kv_lens = [x[1] for x in seq_lens]
num_query_heads = num_heads[0]
num_kv_heads = num_heads[1]
assert num_query_heads % num_kv_heads == 0
max_kv_len = max(kv_lens)
scale = head_size**-0.5
query = torch.randn(sum(query_lens), num_query_heads, head_size, dtype=dtype)
key_value_cache = torch.randn(
NUM_BLOCKS, 2, block_size, num_kv_heads, head_size, dtype=dtype
)
key_cache = key_value_cache[:, 0, :, :, :].squeeze(1)
value_cache = key_value_cache[:, 1, :, :, :].squeeze(1)
# Normalize the scale of the key and value caches to mitigate
# numerical instability.
key_cache /= head_size**0.5
value_cache /= head_size**0.5
max_num_blocks_per_seq = (max_kv_len + block_size - 1) // block_size
block_tables = torch.randint(
0, NUM_BLOCKS, (num_seqs, max_num_blocks_per_seq), dtype=torch.int32
)
qo_indptr = [0]
kv_indptr = [0]
kv_indices = []
kv_last_page_lens = []
for i in range(num_seqs):
seq_len = kv_lens[i]
assert seq_len > 0
num_blocks = (seq_len + block_size - 1) // block_size
kv_indices.extend(block_tables[i, :num_blocks])
kv_indptr.append(kv_indptr[-1] + num_blocks)
kv_last_page_len = seq_len % block_size
if kv_last_page_len == 0:
kv_last_page_len = block_size
kv_last_page_lens.append(kv_last_page_len)
qo_indptr.append(qo_indptr[-1] + query_lens[i])
qo_indptr = torch.tensor(qo_indptr, dtype=torch.int32)
kv_indptr = torch.tensor(kv_indptr, dtype=torch.int32)
kv_indices = torch.tensor(kv_indices, dtype=torch.int32)
kv_last_page_lens = torch.tensor(kv_last_page_lens, dtype=torch.int32)
workspace_buffer = torch.empty(128 * 1024 * 1024, dtype=torch.int8)
wrapper = flashinfer.BatchPrefillWithPagedKVCacheWrapper(workspace_buffer, "NHD")
wrapper.plan(
qo_indptr,
kv_indptr,
kv_indices,
kv_last_page_lens,
num_query_heads,
num_kv_heads,
head_size,
block_size,
window_left=sliding_window - 1 if sliding_window is not None else -1,
q_data_type=dtype,
kv_data_type=dtype,
logits_soft_cap=soft_cap,
)
output = wrapper.run(
query,
key_value_cache,
)
ref_output = ref_paged_attn(
query=query,
key_cache=key_cache,
value_cache=value_cache,
query_lens=query_lens,
kv_lens=kv_lens,
block_tables=block_tables,
scale=scale,
soft_cap=soft_cap,
sliding_window=sliding_window,
)
(
torch.testing.assert_close(output, ref_output, atol=5e-2, rtol=1e-2),
f"{torch.max(torch.abs(output - ref_output))}",
)
@pytest.mark.parametrize("seq_lens", [[(1, 132), (5, 18)]])
@pytest.mark.parametrize("num_heads", NUM_HEADS)
@pytest.mark.parametrize("head_size", HEAD_SIZES)
@pytest.mark.parametrize("block_size", BLOCK_SIZES)
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("soft_cap", SOFT_CAPS)
def test_flashinfer_prefill_with_paged_fp8_kv(
seq_lens: list[tuple[int, int]],
num_heads: tuple[int, int],
head_size: int,
dtype: torch.dtype,
block_size: int,
soft_cap: float | None,
) -> None:
pytest.skip("TODO: fix the accuracy issue")
torch.set_default_device("cuda")
set_random_seed(0)
num_seqs = len(seq_lens)
query_lens = [x[0] for x in seq_lens]
kv_lens = [x[1] for x in seq_lens]
num_query_heads = num_heads[0]
num_kv_heads = num_heads[1]
assert num_query_heads % num_kv_heads == 0
max_kv_len = max(kv_lens)
scale = head_size**-0.5
kv_cache_dtype = torch.float8_e4m3fn
query = torch.randn(sum(query_lens), num_query_heads, head_size, dtype=dtype)
NUM_BLOCKS_FP8 = 2048
key_value_cache = torch.randn(
NUM_BLOCKS_FP8, 2, block_size, num_kv_heads, head_size, dtype=dtype
)
key_cache, value_cache = torch.chunk(key_value_cache, 2, dim=1)
key_cache /= head_size**0.5
value_cache /= head_size**0.5
k_scale = key_cache.amax().item() / 448.0
v_scale = value_cache.amax().item() / 448.0
kv_cache_fp8 = torch.cat([key_cache / k_scale, value_cache / v_scale], dim=1).to(
kv_cache_dtype
)
assert kv_cache_fp8.shape == key_value_cache.shape
max_num_blocks_per_seq = (max_kv_len + block_size - 1) // block_size
block_tables = torch.randint(
0, NUM_BLOCKS_FP8, (num_seqs, max_num_blocks_per_seq), dtype=torch.int32
)
qo_indptr = [0]
kv_indptr = [0]
kv_indices = []
kv_last_page_lens = []
for i in range(num_seqs):
seq_len = kv_lens[i]
assert seq_len > 0
num_blocks = (seq_len + block_size - 1) // block_size
kv_indices.extend(block_tables[i, :num_blocks])
kv_indptr.append(kv_indptr[-1] + num_blocks)
kv_last_page_len = seq_len % block_size
if kv_last_page_len == 0:
kv_last_page_len = block_size
kv_last_page_lens.append(kv_last_page_len)
qo_indptr.append(qo_indptr[-1] + query_lens[i])
qo_indptr = torch.tensor(qo_indptr, dtype=torch.int32)
kv_indptr = torch.tensor(kv_indptr, dtype=torch.int32)
kv_indices = torch.tensor(kv_indices, dtype=torch.int32)
kv_last_page_lens = torch.tensor(kv_last_page_lens, dtype=torch.int32)
workspace_buffer = torch.empty(128 * 1024 * 1024, dtype=torch.int8)
wrapper = flashinfer.BatchPrefillWithPagedKVCacheWrapper(workspace_buffer, "NHD")
wrapper.plan(
qo_indptr,
kv_indptr,
kv_indices,
kv_last_page_lens,
num_query_heads,
num_kv_heads,
head_size,
block_size,
q_data_type=dtype,
kv_data_type=kv_cache_dtype,
logits_soft_cap=soft_cap,
)
output = wrapper.run(query, kv_cache_fp8, k_scale=k_scale, v_scale=v_scale)
ref_output = ref_paged_attn(
query=query,
key_cache=key_cache.squeeze(1),
value_cache=value_cache.squeeze(1),
query_lens=query_lens,
kv_lens=kv_lens,
block_tables=block_tables,
scale=scale,
soft_cap=soft_cap,
)
del query
del block_tables
# verify prefill fp8
(
torch.testing.assert_close(output, ref_output, atol=5e-2, rtol=1e-2),
f"{torch.max(torch.abs(output - ref_output))}",
)
@pytest.mark.parametrize("kv_lens", [[1328, 18, 463], [1, 54, 293, 70]])
@pytest.mark.parametrize("num_heads", NUM_HEADS)
@pytest.mark.parametrize("head_size", HEAD_SIZES)
@pytest.mark.parametrize("block_size", BLOCK_SIZES)
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("soft_cap", SOFT_CAPS)
@pytest.mark.skip(reason="TODO: fix the accuracy issue")
@torch.inference_mode
def test_flashinfer_decode_with_paged_fp8_kv(
kv_lens: list[int],
num_heads: tuple[int, int],
head_size: int,
dtype: torch.dtype,
block_size: int,
soft_cap: float | None,
) -> None:
# test doesn't work for num_heads = (16,16)
torch.set_default_device("cuda")
set_random_seed(0)
num_seqs = len(kv_lens)
num_query_heads = num_heads[0]
num_kv_heads = num_heads[1]
assert num_query_heads % num_kv_heads == 0
max_kv_len = max(kv_lens)
scale = head_size**-0.5
use_tensor_cores = True
kv_cache_dtype = torch.float8_e4m3fn
query = torch.randn(num_seqs, num_query_heads, head_size, dtype=dtype)
NUM_BLOCKS_FP8 = 2048
key_value_cache = torch.randn(
NUM_BLOCKS_FP8, 2, block_size, num_kv_heads, head_size, dtype=dtype
)
key_cache, value_cache = torch.chunk(key_value_cache, 2, dim=1)
key_cache /= head_size**0.5
value_cache /= head_size**0.5
k_scale = key_cache.amax().item() / 448.0
v_scale = value_cache.amax().item() / 448.0
key_cache_fp8 = (key_cache / k_scale).to(kv_cache_dtype)
value_cache_fp8 = (value_cache / v_scale).to(kv_cache_dtype)
assert key_cache_fp8.shape[1] == 1 and value_cache_fp8.shape[1] == 1
kv_cache_fp8 = torch.cat([key_cache_fp8, value_cache_fp8], dim=1)
max_num_blocks_per_seq = (max_kv_len + block_size - 1) // block_size
block_tables = torch.randint(
0, NUM_BLOCKS_FP8, (num_seqs, max_num_blocks_per_seq), dtype=torch.int32
)
kv_indptr = [0]
kv_indices = []
kv_last_page_lens = []
for i in range(num_seqs):
seq_len = kv_lens[i]
assert seq_len > 0
num_blocks = (seq_len + block_size - 1) // block_size
kv_indices.extend(block_tables[i, :num_blocks])
kv_indptr.append(kv_indptr[-1] + num_blocks)
kv_last_page_len = seq_len % block_size
if kv_last_page_len == 0:
kv_last_page_len = block_size
kv_last_page_lens.append(kv_last_page_len)
kv_indptr = torch.tensor(kv_indptr, dtype=torch.int32)
kv_indices = torch.tensor(kv_indices, dtype=torch.int32)
kv_last_page_lens = torch.tensor(kv_last_page_lens, dtype=torch.int32)
workspace_buffer = torch.empty(128 * 1024 * 1024, dtype=torch.int8)
wrapper = flashinfer.BatchDecodeWithPagedKVCacheWrapper(
workspace_buffer, "NHD", use_tensor_cores=use_tensor_cores
)
wrapper.plan(
kv_indptr,
kv_indices,
kv_last_page_lens,
num_query_heads,
num_kv_heads,
head_size,
block_size,
"NONE",
q_data_type=dtype,
kv_data_type=kv_cache_dtype,
logits_soft_cap=soft_cap,
)
output = wrapper.run(query, kv_cache_fp8, k_scale=k_scale, v_scale=v_scale)
key_cache = key_value_cache[:, 0, :, :, :].squeeze(1)
value_cache = key_value_cache[:, 1, :, :, :].squeeze(1)
ref_output = ref_paged_attn(
query=query,
key_cache=key_cache,
value_cache=value_cache,
query_lens=[1] * num_seqs,
kv_lens=kv_lens,
block_tables=block_tables,
scale=scale,
soft_cap=soft_cap,
)
# Temporary fix: Increasing the tolerance. Seems like a flashinfer issue
(
torch.testing.assert_close(output, ref_output, atol=2e-2, rtol=1e-2),
f"{torch.max(torch.abs(output - ref_output))}",
)
@pytest.mark.skipif(
not any(current_platform.is_device_capability_family(f) for f in (80, 90, 120)),
reason="NVFP4 fa2 path",
)
@pytest.mark.parametrize("layout", ["LBNHC", "LBHNC", "BLNHC", "BLHNC"])
def test_flashinfer_nvfp4_kv_cache_update(default_vllm_config, layout):
from vllm.v1.attention.backends import flashinfer as fi
torch.set_default_device("cuda")
order = KVCacheLayout[layout].stride_order
# [L, B, 2H, N, 64 + 8] NVFP4 pages for two layers; layer 1 is written.
shape = (2, 6, 4, 16, 64 + 8)
layers = torch.zeros([shape[i] for i in order], dtype=torch.uint8)
layers = layers.permute(*(order.index(i) for i in range(5)))
one = torch.ones(())
layer = torch.nn.Module()
layer._k_scale = layer._v_scale = one
kv = torch.randn(2, 96, 2, 128, dtype=torch.bfloat16)
impl = fi.FlashInferImpl(4, 128, 1.0, 2, None, None, "nvfp4")
impl.do_kv_cache_update(layer, kv[0], kv[1], layers[1], torch.arange(96))
data, scales, kv_layout = fi._nvfp4_kv_views(layers[1], 2)
pages = torch.arange(6, dtype=torch.int32)[None]
seq_lens = torch.tensor([96], dtype=torch.int32)
deq = torch.empty_like(kv)[:, None]
flashinfer.nvfp4_kv_dequantize_paged(
data, scales, pages, seq_lens, one, one, *deq, kv_layout
)
assert ((deq[:, 0] - kv).norm() / kv.norm()) < 0.2
assert not layers[0].any()
@@ -1,389 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
import torch.nn.functional as F
from torch import Tensor
from vllm.platforms import current_platform
FLASHINFER_WORKSPACE_BUFFER_SIZE = 128 * 1024 * 1024
if not current_platform.is_cuda() or not current_platform.has_device_capability(90):
pytest.skip(
reason="FlashInfer MLA requires CUDA compute capability 9.0 or above.",
allow_module_level=True,
)
else:
from flashinfer.decode import trtllm_batch_decode_with_kv_cache_mla
from flashinfer.mla import BatchMLAPagedAttentionWrapper
requires_sm90 = pytest.mark.skipif(
not current_platform.is_device_capability_family(90),
reason="This test requires an SM90 GPU.",
)
requires_sm10x = pytest.mark.skipif(
not current_platform.is_device_capability_family(100),
reason="This test requires an SM10x GPU.",
)
# Deepseek R1 MLA config.
NUM_HEADS = 128
KV_LORA_RANK = 512
QK_NOPE_HEAD_DIM = 128
QK_ROPE_HEAD_DIM = 64
QK_HEAD_DIM = KV_LORA_RANK + QK_ROPE_HEAD_DIM
SCALE = (QK_NOPE_HEAD_DIM + QK_ROPE_HEAD_DIM) ** -0.5
def _make_decode_inputs(bs: int, block_size: int, dtype: torch.dtype):
"""Build valid trtllm MLA decode inputs on the current CUDA device."""
max_seq_len_cap = 1024
seq_lens = [torch.randint(2, max_seq_len_cap, (1,)).item() for _ in range(bs)]
seq_lens[-1] = max_seq_len_cap
max_seq_len = max(seq_lens)
seq_lens_tensor = torch.tensor(seq_lens, dtype=torch.int32)
# Generate block tables with random but unique block IDs
# From https://github.com/flashinfer-ai/flashinfer/pull/1222
blocks_per_seq = (seq_lens_tensor + block_size - 1) // block_size
max_num_blocks_per_seq = max(blocks_per_seq.max().item(), 4)
total_blocks_needed = int(sum(blocks_per_seq))
all_block_ids = torch.randperm(total_blocks_needed)
block_tables = torch.zeros((bs, max_num_blocks_per_seq), dtype=torch.int32)
block_id = 0
for i in range(bs):
num_blocks_needed = blocks_per_seq[i]
block_tables[i, :num_blocks_needed] = all_block_ids[
block_id : block_id + num_blocks_needed
]
block_id += num_blocks_needed
kv_cache = torch.randn(block_tables.numel(), block_size, QK_HEAD_DIM).to(dtype)
q = torch.randn(bs, NUM_HEADS, QK_HEAD_DIM).to(dtype)
return q, kv_cache, block_tables, seq_lens_tensor, max_seq_len
def ref_mla(
out: Tensor, # (bs, num_heads, v_head_dim)
query: Tensor, # (bs, num_heads, head_dim)
kv_cache: Tensor, # (num_blocks, block_size, head_dim)
scale: float,
block_tables: Tensor, # (bs, max_num_blocks)
seq_lens: Tensor, # (bs,)
):
bs, num_heads, v_head_dim = out.shape
head_dim = query.shape[2]
for i in range(bs):
# gather and flatten KV-cache
kv = kv_cache[block_tables[i]] # (max_num_blocks, block_size, head_dim)
kv = kv.view(1, -1, head_dim)[:, : seq_lens[i]] # (1, seq_len, head_dim)
v = kv[:, :, :v_head_dim]
q = query[i].view(num_heads, 1, head_dim)
o = F.scaled_dot_product_attention(q, kv, v, scale=scale, enable_gqa=True)
out[i] = o.view(num_heads, v_head_dim)
return out
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@pytest.mark.parametrize("bs", [1, 2, 4, 16])
@pytest.mark.parametrize("block_size", [32, 64])
@requires_sm10x
def test_flashinfer_mla_decode(dtype: torch.dtype, bs: int, block_size: int):
torch.set_default_device("cuda")
torch.manual_seed(42)
q, kv_cache, block_tables, seq_lens_tensor, max_seq_len = _make_decode_inputs(
bs, block_size, dtype
)
out_ref = q.new_zeros(bs, NUM_HEADS, KV_LORA_RANK)
ref_mla(out_ref, q, kv_cache, SCALE, block_tables, seq_lens_tensor)
workspace_buffer = torch.zeros(
FLASHINFER_WORKSPACE_BUFFER_SIZE,
dtype=torch.uint8,
device=q.device,
)
# Flashinfer MLA expects the query to be of shape
# (bs, q_len_per_request, num_heads, qk_head_dim),
# where q_len_per_request is the MTP query length (=1 without MTP)
q = q.unsqueeze(1)
out_ans = trtllm_batch_decode_with_kv_cache_mla(
query=q,
kv_cache=kv_cache.unsqueeze(1),
workspace_buffer=workspace_buffer,
qk_nope_head_dim=QK_NOPE_HEAD_DIM,
kv_lora_rank=KV_LORA_RANK,
qk_rope_head_dim=QK_ROPE_HEAD_DIM,
block_tables=block_tables,
seq_lens=seq_lens_tensor,
max_seq_len=max_seq_len,
bmm1_scale=SCALE,
)
out_ans = out_ans.squeeze(1)
torch.testing.assert_close(out_ans, out_ref, atol=1e-2, rtol=1e-2)
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float8_e4m3fn])
@pytest.mark.parametrize("mode", ["tp", "dcp", "hisparse"])
@requires_sm10x
def test_flashinfer_trtllm_sparse_mla_decode_without_rope(dtype, mode, monkeypatch):
"""TP skips preparation; empty DCP contributions and HiSparse outputs vanish."""
import vllm.v1.attention.backends.mla.flashinfer_mla_sparse as backend
from vllm.v1.attention.backends.mla.index_group import HiSparseMLAIndexGroup
torch.set_default_device("cuda")
torch.manual_seed(42)
batch_size, num_heads, block_size, num_blocks, sparse_topk = 3, 8, 64, 4, 2176
valid_lens = [1 if mode == "tp" else 0, 17, 73]
counts = torch.tensor(valid_lens, dtype=torch.int32)
query = torch.randn(batch_size, 1, num_heads, KV_LORA_RANK).to(dtype)
kv_cache = torch.randn(num_blocks, block_size, KV_LORA_RANK).to(dtype)
num_slots = num_blocks * block_size
indices = torch.full((batch_size, sparse_topk), -1, dtype=torch.int32)
for token, length in enumerate(valid_lens):
indices[token, :length] = torch.randperm(num_slots)[:length].int()
empty = counts == 0
reference_indices = indices.clone()
reference_indices[:, 0].masked_fill_(empty, 0)
workspace = torch.empty(FLASHINFER_WORKSPACE_BUFFER_SIZE, dtype=torch.int8)
impl = object.__new__(backend.FlashInferMLASparseImpl)
impl.qk_nope_head_dim = QK_NOPE_HEAD_DIM
impl.kv_lora_rank = KV_LORA_RANK
impl.qk_rope_head_dim = 0
impl.index_group = (
object.__new__(HiSparseMLAIndexGroup) if mode == "hisparse" else None
)
impl._workspace_buffer = workspace
impl.bmm1_scale = QK_NOPE_HEAD_DIM**-0.5
impl.bmm2_scale = 1.0
impl.is_nope_mla = True
impl.need_to_return_lse_for_decode = mode == "dcp"
if mode == "tp":
def unexpected_helper(*args, **kwargs):
pytest.fail("TP must not launch empty-query preparation")
monkeypatch.setattr(
backend, "prepare_sparse_mla_safe_lengths", unexpected_helper
)
out, lse = impl._forward_mqa_kernel(
query.squeeze(1),
kv_cache,
indices,
counts,
layer=None,
block_size=block_size,
is_decode=True,
)
reference = trtllm_batch_decode_with_kv_cache_mla(
query=query,
kv_cache=kv_cache.unsqueeze(1),
workspace_buffer=workspace,
qk_nope_head_dim=QK_NOPE_HEAD_DIM,
kv_lora_rank=KV_LORA_RANK,
qk_rope_head_dim=0,
block_tables=reference_indices.unsqueeze(1),
seq_lens=counts,
max_seq_len=sparse_topk,
sparse_mla_top_k=sparse_topk,
sparse_mla_top_k_lens=counts.clamp(min=1),
bmm1_scale=QK_NOPE_HEAD_DIM**-0.5,
bmm2_scale=1.0,
return_lse=mode == "dcp",
)
if mode == "dcp":
reference, reference_lse = reference
reference_lse = reference_lse.reshape(batch_size, num_heads)
reference_lse.masked_fill_(empty[:, None], float("-inf"))
torch.testing.assert_close(lse, reference_lse, atol=0, rtol=0)
from vllm.v1.attention.ops.dcp import CPTritonContext, correct_attn_out
# The peer owns only our empty selections. Check our contribution after
# the production merge, without zeroing the local attention output.
peer_lse = torch.full_like(lse, float("-inf")).masked_fill(empty[:, None], 0)
out, _ = correct_attn_out(
out, torch.stack((lse, peer_lse)), cp_rank=0, ctx=CPTritonContext()
)
else:
assert lse is None
reference = reference.squeeze(1).masked_fill(empty[:, None, None], 0)
torch.testing.assert_close(out, reference, atol=0, rtol=0)
if dtype == torch.bfloat16:
flat_cache = kv_cache.view(num_slots, KV_LORA_RANK).float()
refs = []
for token, length in enumerate(valid_lens):
selected_kv = flat_cache[reference_indices[token, :length].long()]
scores = torch.einsum("hd,kd->hk", query[token, 0].float(), selected_kv)
probs = torch.softmax(scores * QK_NOPE_HEAD_DIM**-0.5, dim=-1)
refs.append(torch.einsum("hk,kd->hd", probs, selected_kv))
ref = torch.stack(refs).to(torch.bfloat16)
torch.testing.assert_close(out, ref, atol=2e-2, rtol=2e-2)
@requires_sm10x
def test_sparse_mla_safe_lengths():
from vllm.v1.attention.backends.mla.sparse_utils import (
prepare_sparse_mla_safe_lengths,
)
device = "cuda"
num_tokens = 259
# Include a partial tile and expose accidental writes into adjacent storage.
counts_storage = torch.full((num_tokens, 2), 17, dtype=torch.int32, device=device)
counts = counts_storage[:, 0]
counts[::3] = 0
original_counts = counts_storage.clone()
empty = counts == 0
indices_storage = torch.full(
(num_tokens, 256), 42, dtype=torch.int32, device=device
)
indices = indices_storage[:, ::2]
expected_indices = indices_storage.clone()
expected_indices[:, 0].masked_fill_(empty, 0)
safe = prepare_sparse_mla_safe_lengths(indices, counts)
torch.testing.assert_close(safe, counts.clamp(min=1), rtol=0, atol=0)
torch.testing.assert_close(indices_storage, expected_indices, rtol=0, atol=0)
torch.testing.assert_close(counts_storage, original_counts, rtol=0, atol=0)
@requires_sm90
def test_flashinfer_sm90_fp8_mla_decode_without_rope():
"""Hopper FA3 supports BF16 queries over an FP8 cache without KPE."""
torch.manual_seed(42)
device = torch.device("cuda")
batch_size = 2
num_heads = 16
page_size = 16
num_pages = 6
q_nope = torch.randn(
batch_size,
num_heads,
KV_LORA_RANK,
dtype=torch.bfloat16,
device=device,
)
q_pe = torch.empty(
batch_size,
num_heads,
0,
dtype=torch.bfloat16,
device=device,
)
ckv = torch.randn(
num_pages,
page_size,
KV_LORA_RANK,
device=device,
)
fp8_max = torch.finfo(torch.float8_e4m3fn).max
ckv_scale = ckv.abs().max().item() / fp8_max
ckv_fp8 = (ckv / ckv_scale).clamp(-fp8_max, fp8_max).to(torch.float8_e4m3fn)
scale_bf16 = torch.tensor(ckv_scale, dtype=torch.bfloat16, device=device)
ckv_ref = ckv_fp8.to(torch.bfloat16) * scale_bf16
kpe_fp8 = torch.empty(
num_pages,
page_size,
0,
dtype=torch.float8_e4m3fn,
device=device,
)
kpe_ref = torch.empty(
num_pages,
page_size,
0,
dtype=torch.bfloat16,
device=device,
)
qo_indptr = torch.tensor([0, 1, 2], dtype=torch.int32, device=device)
kv_indptr = torch.tensor([0, 3, 5], dtype=torch.int32, device=device)
kv_indices = torch.tensor([4, 1, 3, 0, 5], dtype=torch.int32, device=device)
kv_lens = torch.tensor([45, 29], dtype=torch.int32, device=device)
sm_scale = QK_NOPE_HEAD_DIM**-0.5
def run(
ckv_cache: torch.Tensor,
kpe_cache: torch.Tensor,
**kwargs,
) -> torch.Tensor:
workspace = torch.empty(
FLASHINFER_WORKSPACE_BUFFER_SIZE,
dtype=torch.uint8,
device=device,
)
wrapper = BatchMLAPagedAttentionWrapper(workspace, backend="fa3")
wrapper.plan(
qo_indptr,
kv_indptr,
kv_indices,
kv_lens,
num_heads,
KV_LORA_RANK,
0,
page_size,
False,
sm_scale,
q_data_type=torch.bfloat16,
kv_data_type=ckv_cache.dtype,
)
return wrapper.run(q_nope, q_pe, ckv_cache, kpe_cache, **kwargs)
out_ref = run(ckv_ref, kpe_ref)
out = run(ckv_fp8, kpe_fp8, ckv_scale=ckv_scale, kpe_scale=1.0)
torch.testing.assert_close(out, out_ref, atol=2e-2, rtol=2e-2)
@requires_sm10x
def test_flashinfer_mla_decode_workspace_supports_autotune():
"""VLLM's FlashInfer MLA decode workspace must be int8 for autotuning.
Model Runner V2's warmup autotunes ``trtllm_batch_decode_mla``, which makes
the FlashInfer autotuner enumerate the CuteDSL tactic. That tactic asserts
``workspace_buffer.dtype == torch.int8``; the trtllm-gen path (used for
normal, non-autotuned inference) instead views the buffer as uint8, so a
uint8 workspace only fails once the autotuner tries CuteDSL. That regressed
every DeepSeek MLA test on Blackwell under V2 with
``workspace_buffer must be torch.int8`` (vllm-project/vllm#46646).
"""
from flashinfer.autotuner import autotune
from vllm.v1.attention.backends.mla.flashinfer_mla import _get_workspace_buffer
torch.set_default_device("cuda")
torch.manual_seed(0)
workspace_buffer = _get_workspace_buffer(return_lse=False)
assert workspace_buffer.dtype == torch.int8
q, kv_cache, block_tables, seq_lens_tensor, max_seq_len = _make_decode_inputs(
bs=1, block_size=64, dtype=torch.bfloat16
)
# Under the autotuner the CuteDSL tactic is instantiated with our workspace;
# a uint8 buffer raises AssertionError here, an int8 buffer succeeds.
with torch.inference_mode(), autotune(True):
trtllm_batch_decode_with_kv_cache_mla(
query=q.unsqueeze(1),
kv_cache=kv_cache.unsqueeze(1),
workspace_buffer=workspace_buffer,
qk_nope_head_dim=QK_NOPE_HEAD_DIM,
kv_lora_rank=KV_LORA_RANK,
qk_rope_head_dim=QK_ROPE_HEAD_DIM,
block_tables=block_tables,
seq_lens=seq_lens_tensor,
max_seq_len=max_seq_len,
bmm1_scale=SCALE,
)
@@ -1,842 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
@pytest.mark.parametrize("sm120", [False, True])
def test_deepseek_v4_c128a_adaptive_width_has_capture_stable_stride(
monkeypatch: pytest.MonkeyPatch,
sm120: bool,
):
from vllm.models.deepseek_v4 import sparse_mla
from vllm.platforms.interface import DeviceCapability
monkeypatch.setattr(
sparse_mla.current_platform,
"get_device_capability",
lambda: DeviceCapability(12, 0) if sm120 else DeviceCapability(10, 0),
)
device = torch.device("cuda")
capacity_width = 512
global_decode_buffer = torch.empty(
(2, capacity_width), dtype=torch.int32, device=device
)
prefill_buffer = torch.empty_like(global_decode_buffer)
kwargs = dict(
positions=torch.tensor([255, 511, 383, 639], device=device),
compress_ratio=128,
num_decode_tokens=2,
token_to_req_indices=torch.tensor(
[0, 1, 0, 1], dtype=torch.int32, device=device
),
block_table=torch.tensor([[3], [5]], dtype=torch.int32, device=device),
block_size=capacity_width,
slot_mapping=torch.arange(4, dtype=torch.int64, device=device),
global_decode_buffer=global_decode_buffer,
decode_lens_buffer=torch.empty(2, dtype=torch.int32, device=device),
prefill_buffer=prefill_buffer,
)
captured_decode, _, captured_prefill = sparse_mla.build_c128a_topk_metadata(
max_compressed_tokens=256,
**kwargs,
)
# SM120 keeps the decode view contiguous across the full buffer width;
# other backends get the active-width slice. The prefill view is always
# narrowed.
expected_decode_width = capacity_width if sm120 else 256
assert captured_decode.shape == (2, expected_decode_width)
assert captured_prefill.shape == (2, 256)
assert captured_decode.stride(0) == captured_prefill.stride(0) == capacity_width
assert captured_decode.is_contiguous() == sm120
captured_rows = torch.empty((4, 4), dtype=torch.int32, device=device)
captured_rows[:2].copy_(captured_decode[:, :4])
captured_rows[2:].copy_(captured_prefill[:, :4])
torch.accelerator.synchronize()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
captured_rows[:2].copy_(captured_decode[:, :4])
captured_rows[2:].copy_(captured_prefill[:, :4])
global_decode_buffer.fill_(-99)
prefill_buffer.fill_(-99)
sparse_mla.build_c128a_topk_metadata(
max_compressed_tokens=128,
**kwargs,
)
graph.replay()
assert captured_rows.cpu().tolist() == [
[1536, 1537, -1, -1],
[2560, 2561, 2562, 2563],
[0, 1, 2, -1],
[0, 1, 2, 3],
]
assert torch.all(global_decode_buffer[:, 128:] == -99)
assert torch.all(prefill_buffer[:, 128:] == -99)
def test_sparse_flashmla_metadata_smoke():
import vllm.v1.attention.ops.flashmla as fm
ok, reason = fm.is_flashmla_sparse_supported()
if not ok:
pytest.skip(reason)
device = torch.device("cuda")
batch_size = 1
seqlen_q = 1
num_heads_q = 128
num_heads_k = 1
q_seq_per_hk = seqlen_q * num_heads_q // num_heads_k
topk = 128
cache_seqlens = torch.zeros(batch_size, dtype=torch.int32, device=device)
tile_md, num_splits = fm.get_mla_metadata(
cache_seqlens,
q_seq_per_hk,
num_heads_k,
num_heads_q=num_heads_q,
topk=topk,
is_fp8_kvcache=True,
)
assert isinstance(tile_md, fm.FlashMLASchedMeta)
assert tile_md.tile_scheduler_metadata is None
assert tile_md.num_splits is None
assert num_splits is None
def test_sparse_flashmla_decode_matches_cache_writer_scales():
import vllm.v1.attention.ops.flashmla as fm
from vllm import _custom_ops as ops
ok, reason = fm.is_flashmla_sparse_supported()
if not ok:
pytest.skip(reason)
device = torch.device("cuda")
batch_size = 1
seqlen_q = 1
num_heads_q = 64
head_dim_k = 576
head_dim_v = 512
num_heads_k = 1
page_block_size = 64
bytes_per_token = 656
topk = 128
# Metadata
q_seq_per_hk = seqlen_q * num_heads_q // num_heads_k
# q_heads_per_hk = num_heads_q // num_heads_k
cache_seqlens = torch.ones(batch_size, dtype=torch.int32, device=device)
tile_md, num_splits = fm.get_mla_metadata(
cache_seqlens,
q_seq_per_hk,
num_heads_k,
num_heads_q=num_heads_q,
topk=topk,
is_fp8_kvcache=True,
)
# Inputs
q = torch.zeros(
(batch_size, seqlen_q, num_heads_q, head_dim_k),
dtype=torch.bfloat16,
device=device,
)
cache_rows = torch.zeros(
(1, page_block_size, bytes_per_token), dtype=torch.uint8, device=device
)
# 336 / 448 = 0.75 distinguishes arbitrary FP32 from power-of-two scales.
kv_c = torch.full((1, 512), 336.0, dtype=torch.bfloat16, device=device)
kv_c[:, -128:] = 0
ops.concat_and_cache_mla(
kv_c,
torch.zeros((1, 64), dtype=torch.bfloat16, device=device),
cache_rows,
torch.zeros(1, dtype=torch.int64, device=device),
"fp8_ds_mla",
torch.ones(1, dtype=torch.float32, device=device),
)
scales = cache_rows[0, 0].view(torch.float32)[128:132]
expected_scales = torch.tensor([1.0, 1.0, 1.0, 2**-13], device=device)
torch.testing.assert_close(scales, expected_scales, rtol=0, atol=0)
cached_nope = cache_rows[0, 0, :512].view(torch.float8_e4m3fn).float()
expected = (cached_nope * scales.repeat_interleave(128)).to(torch.bfloat16)
k_cache = cache_rows.unsqueeze(2)
indices = torch.full(
(batch_size, seqlen_q, topk), -1, dtype=torch.int32, device=device
)
indices[..., 0] = 0
# SM90 sparse decode only supports a fixed top-k width. Padding with -1
# exercises the same cache data on SM90 and SM100 without topk_length.
block_table = torch.zeros((batch_size, 128), dtype=torch.int32, device=device)
out, lse = fm.flash_mla_with_kvcache(
q,
k_cache,
block_table,
cache_seqlens,
head_dim_v,
tile_md,
num_splits,
indices=indices,
is_fp8_kvcache=True,
)
assert out.shape[0] == batch_size
assert out.shape[-1] == head_dim_v
assert lse.shape[0] == batch_size
torch.testing.assert_close(
out, expected.view(1, 1, 1, 512).expand_as(out), rtol=0, atol=0
)
@pytest.mark.parametrize("h_q", [64, 128])
def test_sparse_flashmla_prefill_smoke(h_q: int):
import vllm.v1.attention.ops.flashmla as fm
ok, reason = fm.is_flashmla_sparse_supported()
if not ok:
pytest.skip(reason)
device = torch.device("cuda")
torch.manual_seed(0)
s_q = 1
s_kv = 8
h_kv = 1
d_qk = 576
d_v = 512
topk = 128
q = torch.randn((s_q, h_q, d_qk), dtype=torch.bfloat16, device=device)
kv = torch.randn((s_kv, h_kv, d_qk), dtype=torch.bfloat16, device=device)
indices = torch.randint(s_kv, (s_q, h_kv, topk), dtype=torch.int32, device=device)
reference_indices = indices.clone()
reference_indices[..., 1:] = -1
kwargs = {"topk_length": torch.ones(1, dtype=torch.int32, device=device)}
reference = fm.flash_mla_sparse_fwd(q, kv, reference_indices, 1.0, d_v, **kwargs)
actual = fm.flash_mla_sparse_fwd(q, kv, indices, 1.0, d_v, **kwargs)
for actual_tensor, reference_tensor in zip(actual, reference):
torch.testing.assert_close(actual_tensor, reference_tensor, rtol=0, atol=0)
assert actual[0].shape == (s_q, h_q, d_v)
def test_deepseek_v4_prefill_chunk_planning_expands_for_short_sequences():
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
metadata = DeepseekSparseSWAMetadata(
block_table=torch.empty(0, dtype=torch.int32),
slot_mapping=torch.empty(0, dtype=torch.int32),
block_size=64,
num_prefills=5,
prefill_seq_lens_cpu=torch.tensor([80, 96, 112, 128, 144], dtype=torch.int32),
prefill_query_lens_cpu=torch.tensor([4, 4, 4, 4, 4], dtype=torch.int32),
prefill_window_size=64,
prefill_max_model_len=1024,
prefill_max_num_batched_tokens=128,
)
chunk_plan = metadata.get_prefill_chunk_plan(compress_ratio=4, prefill_chunk_size=4)
# the adaptive plan keeps all 5 in one chunk
assert chunk_plan == [(0, 5, 36, 103)]
def test_flashinfer_sparse_indices_cache(monkeypatch):
from vllm.models.deepseek_v4.nvidia import flashinfer_sparse as flashinfer_mod
from vllm.models.deepseek_v4.sparse_mla import DeepseekV4FlashMLAMetadata
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
builder_calls = 0
def fake_build(*args, **kwargs):
nonlocal builder_calls
builder_calls += 1
return (
torch.tensor([[builder_calls]], dtype=torch.int32),
torch.tensor([builder_calls], dtype=torch.int32),
)
monkeypatch.setattr(
flashinfer_mod, "build_flashinfer_mixed_sparse_indices", fake_build
)
def make_attn(compress_ratio: int, topk_width: int):
attn = object.__new__(flashinfer_mod.DeepseekV4FlashInferMLAAttention)
attn.compress_ratio = compress_ratio
attn.window_size = 4
attn.topk_indices_buffer = torch.tensor(
[[0, 1], [2, 3], [4, 5]], dtype=torch.int32
)[:, :topk_width]
return attn
def make_swa_metadata():
return DeepseekSparseSWAMetadata(
block_table=torch.tensor([[0, 1], [2, 3]], dtype=torch.int32),
slot_mapping=torch.tensor([0, 1], dtype=torch.int64),
block_size=64,
seq_lens=torch.tensor([8, 10], dtype=torch.int32),
query_start_loc=torch.tensor([0, 1, 3], dtype=torch.int32),
query_start_loc_cpu=torch.tensor([0, 1, 3], dtype=torch.int32),
token_to_req_indices=torch.tensor([0, 1, 1], dtype=torch.int32),
decode_swa_indices=torch.tensor([[5, 6, -1, -1]], dtype=torch.int32),
decode_swa_lens=torch.tensor([2], dtype=torch.int32),
decode_swa_width=4,
is_valid_token=torch.tensor([True], dtype=torch.bool),
num_decodes=1,
num_prefills=1,
num_decode_tokens=1,
num_prefill_tokens=2,
)
def make_flashmla_metadata():
return DeepseekV4FlashMLAMetadata(
num_reqs=2,
max_query_len=2,
max_seq_len=10,
num_actual_tokens=3,
query_start_loc=torch.tensor([0, 1, 3], dtype=torch.int32),
slot_mapping=torch.tensor([0, 1, 2], dtype=torch.int64),
block_table=torch.tensor([[0, 1], [2, 3]], dtype=torch.int32),
req_id_per_token=torch.tensor([0, 1, 1], dtype=torch.int32),
block_size=256,
topk_tokens=2,
c128a_global_decode_topk_indices=torch.tensor(
[[[9, 10]]], dtype=torch.int32
),
c128a_decode_topk_lens=torch.tensor([2], dtype=torch.int32),
c128a_prefill_topk_indices=torch.tensor(
[[0, 1], [1, 2]], dtype=torch.int32
),
)
swa_attn = make_attn(1, 0)
swa_metadata = make_swa_metadata()
_, _, sparse_indices_first, sparse_lens_first = (
swa_attn._build_sparse_index_metadata(
kv_cache=None,
swa_k_cache=torch.empty((1, 64, 512), dtype=torch.bfloat16),
swa_metadata=swa_metadata,
attn_metadata=None,
swa_only=True,
)
)
_, _, sparse_indices_second, sparse_lens_second = (
swa_attn._build_sparse_index_metadata(
kv_cache=None,
swa_k_cache=torch.empty((1, 64, 512), dtype=torch.bfloat16),
swa_metadata=swa_metadata,
attn_metadata=None,
swa_only=True,
)
)
assert builder_calls == 1
assert sparse_indices_first is sparse_indices_second
assert sparse_lens_first is sparse_lens_second
c128a_attn = make_attn(128, 2)
c128a_metadata = make_swa_metadata()
c128a_flashmla_md = make_flashmla_metadata()
_, _, sparse_indices_first, sparse_lens_first = (
c128a_attn._build_sparse_index_metadata(
kv_cache=torch.empty((1, 2, 512), dtype=torch.bfloat16),
swa_k_cache=torch.empty((1, 64, 512), dtype=torch.bfloat16),
swa_metadata=c128a_metadata,
attn_metadata=c128a_flashmla_md,
swa_only=False,
)
)
_, _, sparse_indices_second, sparse_lens_second = (
c128a_attn._build_sparse_index_metadata(
kv_cache=torch.empty((1, 2, 512), dtype=torch.bfloat16),
swa_k_cache=torch.empty((1, 64, 512), dtype=torch.bfloat16),
swa_metadata=c128a_metadata,
attn_metadata=c128a_flashmla_md,
swa_only=False,
)
)
assert builder_calls == 2
assert sparse_indices_first is sparse_indices_second
assert sparse_lens_first is sparse_lens_second
c4a_attn = make_attn(4, 2)
c4a_metadata = make_swa_metadata()
c4a_flashmla_md = make_flashmla_metadata()
c4a_flashmla_md.c128a_global_decode_topk_indices = None
c4a_flashmla_md.c128a_decode_topk_lens = None
c4a_flashmla_md.c128a_prefill_topk_indices = None
_, _, sparse_indices_third, sparse_lens_third = (
c4a_attn._build_sparse_index_metadata(
kv_cache=torch.empty((1, 2, 512), dtype=torch.bfloat16),
swa_k_cache=torch.empty((1, 64, 512), dtype=torch.bfloat16),
swa_metadata=c4a_metadata,
attn_metadata=c4a_flashmla_md,
swa_only=False,
)
)
_, _, sparse_indices_fourth, sparse_lens_fourth = (
c4a_attn._build_sparse_index_metadata(
kv_cache=torch.empty((1, 2, 512), dtype=torch.bfloat16),
swa_k_cache=torch.empty((1, 64, 512), dtype=torch.bfloat16),
swa_metadata=c4a_metadata,
attn_metadata=c4a_flashmla_md,
swa_only=False,
)
)
assert builder_calls == 4
assert sparse_indices_third is not sparse_indices_fourth
assert sparse_lens_third is not sparse_lens_fourth
def test_flashinfer_sparse_index_preserves_logical_window(monkeypatch):
from vllm.models.deepseek_v4.nvidia import flashinfer_sparse as flashinfer_mod
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
captured_shapes_and_windows: list[tuple[int, int]] = []
def fake_build(*args, **kwargs):
# window_size is the 12th positional arg of
# build_flashinfer_mixed_sparse_indices.
captured_shapes_and_windows.append((args[0].shape[-1], args[11]))
num_tokens = args[0].shape[0] + args[3].shape[0]
return (
torch.zeros((num_tokens, 1), dtype=torch.int32),
torch.zeros((num_tokens,), dtype=torch.int32),
)
monkeypatch.setattr(
flashinfer_mod, "build_flashinfer_mixed_sparse_indices", fake_build
)
attn = object.__new__(flashinfer_mod.DeepseekV4FlashInferMLAAttention)
attn.compress_ratio = 1
attn.window_size = 4
attn.topk_indices_buffer = torch.zeros((4, 0), dtype=torch.int32)
wide_width = 8
wide_indices = torch.full((1, wide_width), -1, dtype=torch.int32)
wide_indices[0, :2] = torch.tensor([5, 6], dtype=torch.int32)
wide_metadata = DeepseekSparseSWAMetadata(
block_table=torch.tensor([[0, 1]], dtype=torch.int32),
slot_mapping=torch.tensor([0], dtype=torch.int64),
block_size=64,
seq_lens=torch.tensor([8], dtype=torch.int32),
query_start_loc=torch.tensor([0, 1], dtype=torch.int32),
query_start_loc_cpu=torch.tensor([0, 1], dtype=torch.int32),
token_to_req_indices=torch.tensor([0], dtype=torch.int32),
decode_swa_indices=wide_indices,
decode_swa_lens=torch.tensor([2], dtype=torch.int32),
decode_swa_width=wide_width,
is_valid_token=torch.tensor([True], dtype=torch.bool),
num_decodes=1,
num_prefills=0,
num_decode_tokens=1,
num_prefill_tokens=0,
)
attn._build_sparse_index_metadata(
kv_cache=None,
swa_k_cache=torch.empty((1, 64, 512), dtype=torch.bfloat16),
swa_metadata=wide_metadata,
attn_metadata=None,
swa_only=True,
)
assert captured_shapes_and_windows == [(wide_width, attn.window_size)]
empty_width = 8
empty_metadata = DeepseekSparseSWAMetadata(
block_table=torch.tensor([[0, 1]], dtype=torch.int32),
slot_mapping=torch.tensor([0, 1], dtype=torch.int64),
block_size=64,
seq_lens=torch.tensor([8], dtype=torch.int32),
query_start_loc=torch.tensor([0, 2], dtype=torch.int32),
query_start_loc_cpu=torch.tensor([0, 2], dtype=torch.int32),
token_to_req_indices=torch.tensor([0, 0], dtype=torch.int32),
decode_swa_indices=torch.empty((0, 1, empty_width), dtype=torch.int32),
decode_swa_lens=torch.empty((0,), dtype=torch.int32),
decode_swa_width=empty_width,
is_valid_token=torch.tensor([True, True], dtype=torch.bool),
num_decodes=0,
num_prefills=1,
num_decode_tokens=0,
num_prefill_tokens=2,
)
attn._build_sparse_index_metadata(
kv_cache=None,
swa_k_cache=torch.empty((1, 64, 512), dtype=torch.bfloat16),
swa_metadata=empty_metadata,
attn_metadata=None,
swa_only=True,
)
assert captured_shapes_and_windows == [
(wide_width, attn.window_size),
(empty_width, attn.window_size),
]
def test_flashinfer_mixed_sparse_indices_separates_window_and_padded_width():
from vllm.models.deepseek_v4.common.ops.cache_utils import (
build_flashinfer_mixed_sparse_indices,
)
device = torch.device("cuda")
padded_width = 8
logical_window = 4
sparse_indices, sparse_lens = build_flashinfer_mixed_sparse_indices(
decode_swa_indices=torch.empty(
(0, padded_width), dtype=torch.int32, device=device
),
decode_compressed_indices=None,
decode_compressed_topk_lens=None,
prefill_topk_indices=torch.empty((1, 0), dtype=torch.int32, device=device),
query_start_loc=torch.tensor([0, 1], dtype=torch.int32, device=device),
seq_lens=torch.tensor([logical_window], dtype=torch.int32, device=device),
token_to_req_indices=torch.tensor([0], dtype=torch.int32, device=device),
swa_block_table=torch.tensor([[0]], dtype=torch.int32, device=device),
swa_block_size=64,
compressed_block_table=None,
compressed_block_size=64,
window_size=logical_window,
compress_ratio=1,
topk=0,
)
assert sparse_indices.shape == (1, padded_width)
assert sparse_indices[0].cpu().tolist() == [0, 1, 2, 3, -1, -1, -1, -1]
assert sparse_lens.cpu().tolist() == [padded_width]
def test_flashinfer_mixed_sparse_indices_noncausal_rows_have_no_active_gaps():
"""A DSpark non-causal row keeps -1 out of the kernel's active ranges.
Each draft token's causal window holds its first visible entries, the rest
of the block starts at column `window`, and the compressed entries follow
directly; `sparse_topk_lens` ends right after them.
"""
from vllm.models.deepseek_v4.common.ops.cache_utils import (
build_flashinfer_mixed_sparse_indices,
)
device = torch.device("cuda")
window, width = 4, 8
# One request: 1 context token, a 2-token draft block, slots 0..2 visible.
decode_swa = torch.full((2, width), -1, dtype=torch.int32, device=device)
decode_swa[:, :3] = torch.arange(3, dtype=torch.int32, device=device)
compressed = torch.tensor(
[[100, 101, -1, -1]] * 2, dtype=torch.int32, device=device
)
sparse_indices, sparse_lens = build_flashinfer_mixed_sparse_indices(
decode_swa_indices=decode_swa,
decode_compressed_indices=compressed,
decode_compressed_topk_lens=torch.tensor(
[2, 2], dtype=torch.int32, device=device
),
prefill_topk_indices=torch.empty((0, 4), dtype=torch.int32, device=device),
query_start_loc=torch.tensor([0, 2], dtype=torch.int32, device=device),
seq_lens=torch.tensor([3], dtype=torch.int32, device=device),
token_to_req_indices=torch.tensor([0, 0], dtype=torch.int32, device=device),
swa_block_table=torch.tensor([[0]], dtype=torch.int32, device=device),
swa_block_size=64,
compressed_block_table=torch.tensor([[0]], dtype=torch.int32, device=device),
compressed_block_size=64,
window_size=window,
compress_ratio=128,
topk=0,
)
assert sparse_indices.cpu().tolist() == [
# pos 1: window [0, 1]; slot 2 spills to column 4; compressed after it.
[0, 1, -1, -1, 2, 100, 101, -1, -1, -1, -1, -1],
# pos 2: window [0, 1, 2]; nothing spills; compressed at column 4.
[0, 1, 2, -1, 100, 101, -1, -1, -1, -1, -1, -1],
]
assert sparse_lens.cpu().tolist() == [7, 6]
def test_flashinfer_mixed_sparse_indices_noncausal_pad_rows_keep_min_length():
"""A CUDA-graph pad token of the DSpark graph (a zero-length request) gets
a causal pad row's shape instead of negative spill arithmetic."""
from vllm.models.deepseek_v4.common.ops.cache_utils import (
build_flashinfer_mixed_sparse_indices,
)
device = torch.device("cuda")
window, width = 128, 192
decode_swa = torch.full((3, width), -1, dtype=torch.int32, device=device)
decode_swa[:2, :3] = torch.arange(3, dtype=torch.int32, device=device)
sparse_indices, sparse_lens = build_flashinfer_mixed_sparse_indices(
decode_swa_indices=decode_swa,
decode_compressed_indices=None,
decode_compressed_topk_lens=None,
prefill_topk_indices=torch.empty((0, 0), dtype=torch.int32, device=device),
# Request 1 is the padding: no query tokens, seq_len 0.
query_start_loc=torch.tensor([0, 2, 2], dtype=torch.int32, device=device),
seq_lens=torch.tensor([3, 0], dtype=torch.int32, device=device),
token_to_req_indices=torch.tensor([0, 0, 1], dtype=torch.int32, device=device),
swa_block_table=torch.zeros((2, 1), dtype=torch.int32, device=device),
swa_block_size=64,
compressed_block_table=None,
compressed_block_size=64,
window_size=window,
compress_ratio=1,
topk=0,
)
assert sparse_lens.cpu().tolist() == [129, 128, 128]
assert (sparse_indices[2] == -1).all()
@pytest.mark.parametrize("model", ["deepseek_v4", "deepseek_v41"])
@pytest.mark.parametrize("page_padding", [0, 64])
def test_flashinfer_sparse_forward_reads_packed_kv_rows(model, page_padding):
"""Decode and prefill must read storage rows after vLLM remaps packed pages."""
import importlib
from vllm.platforms import current_platform
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
if not current_platform.is_device_capability_family(100):
pytest.skip("Requires FlashInfer TRTLLM sparse MLA on SM100/SM103")
mod = importlib.import_module(f"vllm.models.{model}.nvidia.flashinfer_sparse")
device = "cuda"
block_size = 64
storage = torch.full(
(3, block_size + page_padding, 512), 99, dtype=torch.bfloat16, device=device
)
cache = storage[:, :block_size]
cache[1].fill_(2)
cache[2].fill_(3)
attn = object.__new__(mod.DeepseekV4FlashInferMLAAttention)
attn.compress_ratio = 1
attn.window_size = 128
attn.scale = 512**-0.5
attn.kv_cache_torch_dtype = torch.bfloat16
attn.attn_sink = torch.full((64,), -float("inf"), device=device)
attn.topk_indices_buffer = torch.empty((3, 0), dtype=torch.int32, device=device)
decode_indices = torch.full((1, 128), -1, dtype=torch.int32, device=device)
decode_indices[0, :4] = torch.arange(64, 68, dtype=torch.int32, device=device)
query_start = torch.tensor([0, 1, 3], dtype=torch.int32)
metadata = DeepseekSparseSWAMetadata(
block_table=torch.tensor([[1], [2]], dtype=torch.int32, device=device),
slot_mapping=torch.tensor([67, 130, 131], device=device),
block_size=block_size,
seq_lens=torch.tensor([4, 4], dtype=torch.int32, device=device),
query_start_loc=query_start.to(device),
query_start_loc_cpu=query_start,
token_to_req_indices=torch.tensor([0, 1, 1], dtype=torch.int32, device=device),
decode_swa_indices=decode_indices,
decode_swa_lens=torch.tensor([4], dtype=torch.int32, device=device),
decode_swa_width=128,
is_valid_token=torch.ones(3, dtype=torch.bool, device=device),
replay_start=torch.zeros(2, dtype=torch.int32, device=device),
num_decodes=1,
num_prefills=1,
num_decode_tokens=1,
num_prefill_tokens=2,
max_decode_query_len=1,
)
query = torch.zeros((3, 64, 512), dtype=torch.bfloat16, device=device)
output = torch.empty_like(query)
attn._forward(query, None, cache, metadata, None, True, output)
expected = torch.tensor([2, 3, 3], dtype=output.dtype, device=device)
torch.testing.assert_close(output, expected[:, None, None].expand_as(output))
def _make_rope_quant_attn(mod, num_rows: int, device: str):
"""A bare DSv4 FlashInfer layer set up for RopeQuant: 128 heads, fp8 KV."""
from types import SimpleNamespace
heads, groups, rank = 128, 16, 256
angles = torch.arange(2048, device=device, dtype=torch.float32)[:, None] * 0.05
angles = angles + torch.arange(32, device=device)[None] * 0.1
attn = object.__new__(mod.DeepseekV4FlashInferMLAAttention)
attn.compress_ratio = 1
attn.window_size = 128
attn.scale = 512**-0.5
attn.kv_cache_torch_dtype = torch.float8_e4m3fn
attn._flashinfer_fp8_bmm1_scale = attn.scale
attn._flashinfer_fp8_bmm2_scale = 1.0
attn.attn_sink = torch.linspace(-1, 1, heads, device=device)
attn.padded_heads = heads
attn.topk_indices_buffer = torch.empty(
(num_rows, 0), dtype=torch.int32, device=device
)
attn.rotary_emb = SimpleNamespace(
cos_sin_cache=torch.cat((angles.cos(), angles.sin()), -1).contiguous()
)
attn.n_local_heads, attn.n_local_groups, attn.o_lora_rank = heads, groups, rank
attn.nope_head_dim, attn.rope_head_dim = 448, 64
attn._einsum_recipe, attn._tma_aligned_scales = (1, 1, 128), True
attn.wo_a = SimpleNamespace(
weight=(torch.randn((groups, rank, 4096), device=device) * 0.05).to(
torch.float8_e4m3fn
),
weight_scale=torch.ones((groups, rank, 32), device=device),
)
attn.wo_b = lambda z: z
return attn
def _rope_quant_vs_unfused(attn, q, cache, metadata, positions, num_tokens):
"""Run `_forward` unfused and fused; return (bf16 attention, z_unfused, z_fused)."""
from vllm.models.deepseek_v4.nvidia.ops.o_proj import rope_quant_attn_out
heads, groups = attn.n_local_heads, attn.n_local_groups
unfused = torch.empty(q.shape, dtype=torch.bfloat16, device=q.device)
attn._forward(q, None, cache, metadata, None, True, unfused)
fused = rope_quant_attn_out(q.shape[0], q.device)
metadata.flashinfer_sparse_index_cache.clear()
attn._forward(q, None, cache, metadata, None, True, fused)
z_unfused = attn._o_proj(unfused[:num_tokens, :heads], positions)
z_fused = attn._o_proj(fused, positions)[:num_tokens]
return unfused[:num_tokens], z_unfused, z_fused.view(num_tokens, groups, -1)
def _assert_projects_alike(z_unfused, z_fused):
z_unfused = z_unfused.float().view_as(z_fused)
rel = (z_fused.float() - z_unfused).norm() / z_unfused.norm()
assert rel < 2e-2, rel
def _skip_unless_rope_quant():
from vllm.platforms import current_platform
if not current_platform.is_device_capability_family(100):
pytest.skip("Requires FlashInfer TRTLLM sparse MLA on SM100/SM103")
def test_flashinfer_rope_quant_matches_unfused_o_proj():
"""RopeQuant's fused output must project like the bf16 + inv-RoPE/quant path.
One mixed step (two decodes, a three-token prefill) plus CUDA-graph pad rows,
so both calls run over the full buffer with un-rebased cum_seq_lens_q.
"""
from vllm.models.deepseek_v4.nvidia import flashinfer_sparse as mod
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
_skip_unless_rope_quant()
torch.manual_seed(0)
device = "cuda"
block_size = 64
q_lens, seq_lens = [1, 1, 3], [5, 9, 70]
num_tokens, num_rows = sum(q_lens), sum(q_lens) + 3
cache = torch.randn((3, block_size, 512), device=device).to(torch.float8_e4m3fn)
attn = _make_rope_quant_attn(mod, num_rows, device)
decode_indices = torch.full((2, 128), -1, dtype=torch.int32, device=device)
for i, seq_len in enumerate(seq_lens[:2]):
decode_indices[i, :seq_len] = torch.arange(
i * block_size, i * block_size + seq_len, device=device
)
query_start = torch.tensor([0, 1, 2, 5], dtype=torch.int32)
metadata = DeepseekSparseSWAMetadata(
block_table=torch.tensor([[0], [1], [2]], dtype=torch.int32, device=device),
slot_mapping=torch.zeros(num_tokens, dtype=torch.int64, device=device),
block_size=block_size,
seq_lens=torch.tensor(seq_lens, dtype=torch.int32, device=device),
query_start_loc=query_start.to(device),
query_start_loc_cpu=query_start,
token_to_req_indices=torch.tensor(
[0, 1, 2, 2, 2], dtype=torch.int32, device=device
),
decode_swa_indices=decode_indices,
decode_swa_lens=torch.tensor(seq_lens[:2], dtype=torch.int32, device=device),
decode_swa_width=128,
is_valid_token=torch.ones(num_tokens, dtype=torch.bool, device=device),
num_decodes=2,
num_prefills=1,
num_decode_tokens=2,
num_prefill_tokens=3,
max_decode_query_len=1,
)
positions = torch.tensor([4, 8, 67, 68, 69], device=device)
q = torch.randn((num_rows, 128, 512), device=device).to(torch.float8_e4m3fn)
_, z_unfused, z_fused = _rope_quant_vs_unfused(
attn, q, cache, metadata, positions, num_tokens
)
_assert_projects_alike(z_unfused, z_fused)
@pytest.mark.parametrize("context_len", [0, 20, 127, 900])
def test_flashinfer_dspark_noncausal_block_sees_future_tokens(context_len):
"""Every DSpark draft token attends to the whole block, fused or not.
The kernel counts every entry of its active index ranges as a key, so the
builder must place the block around each query's causal window without -1
gaps, down to an empty context.
"""
from vllm.models.deepseek_v4.nvidia import flashinfer_sparse as mod
from vllm.v1.attention.backends.mla.compressor_utils import (
get_dspark_swa_index_width,
)
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
_skip_unless_rope_quant()
torch.manual_seed(0)
device = "cuda"
block_size, window, draft = 64, 128, 4
width = get_dspark_swa_index_width(window, draft)
contexts = [context_len, context_len + 37]
num_reqs, num_tokens = len(contexts), len(contexts) * draft
num_rows = num_tokens + 2
pages = -(-(max(contexts) + draft) // block_size)
cache = torch.randn((num_reqs * pages, block_size, 512), device=device)
cache = cache.clamp_(-1, 1).to(torch.float8_e4m3fn)
attn = _make_rope_quant_attn(mod, num_rows, device)
attn.attn_sink = torch.full((128,), -float("inf"), device=device)
# As the DSpark SWA builder lays them out: the trailing window of context
# and the whole block, contiguous from column 0.
decode_indices = torch.full((num_tokens, width), -1, dtype=torch.int32)
visible, positions = [], []
for r, ctx in enumerate(contexts):
slots = torch.arange(max(ctx - window, 0), ctx + draft) + r * pages * block_size
for k in range(draft):
decode_indices[r * draft + k, : slots.numel()] = slots.to(torch.int32)
visible.append(slots)
positions.append(ctx + k)
query_start = torch.arange(0, num_tokens + 1, draft, dtype=torch.int32)
seq_lens = torch.tensor([c + draft for c in contexts], dtype=torch.int32)
metadata = DeepseekSparseSWAMetadata(
block_table=torch.arange(
num_reqs * pages, dtype=torch.int32, device=device
).view(num_reqs, pages),
slot_mapping=torch.zeros(num_tokens, dtype=torch.int64, device=device),
block_size=block_size,
seq_lens=seq_lens.to(device),
query_start_loc=query_start.to(device),
query_start_loc_cpu=query_start,
token_to_req_indices=torch.arange(
num_reqs, dtype=torch.int32, device=device
).repeat_interleave(draft),
decode_swa_indices=decode_indices.to(device),
decode_swa_lens=torch.tensor(
[v.numel() for v in visible], dtype=torch.int32, device=device
),
decode_swa_width=width,
is_valid_token=torch.ones(num_tokens, dtype=torch.bool, device=device),
num_decodes=num_reqs,
num_prefills=0,
num_decode_tokens=num_tokens,
num_prefill_tokens=0,
max_decode_query_len=draft,
)
q = torch.randn((num_rows, 128, 512), device=device).clamp_(-1, 1)
q = q.to(torch.float8_e4m3fn)
attention, z_unfused, z_fused = _rope_quant_vs_unfused(
attn, q, cache, metadata, torch.tensor(positions, device=device), num_tokens
)
keys_all = cache.flatten(0, 1).float()
for token, slots in enumerate(visible):
keys = keys_all[slots.to(device)]
weights = torch.softmax(q[token].float() @ keys.T * attn.scale, -1)
torch.testing.assert_close(
attention[token].float(), weights @ keys, atol=0.05, rtol=0.05
)
_assert_projects_alike(z_unfused, z_fused)
@@ -1,310 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""RoPE equivalence tests for the fused Kimi-K3 MLA epilogues."""
import pytest
import torch
from vllm.models.kimi_k3.nvidia.ops.fused_mla_key_concat_kv_cache import (
fused_mla_decode_q_concat_kv_cache_insert,
fused_mla_key_concat_ds_mla_insert,
fused_mla_key_concat_kv_cache_insert,
fused_mla_kv_concat,
fused_mla_kv_concat_quant_fp8,
fused_mla_qkv_quant_kv_cache_fp8_insert,
)
from vllm.platforms import current_platform
pytestmark = pytest.mark.skipif(
not current_platform.is_cuda(), reason="Kimi-K3 fused MLA requires CUDA"
)
_DTYPE = torch.bfloat16
_NUM_TOKENS = 3
_NUM_HEADS = 4
_BLOCK_SIZE = 8
_POSITIONS = (1, 7, 13)
_SLOTS = (0, 3, 9)
def _randn(*shape: int, dtype: torch.dtype = _DTYPE) -> torch.Tensor:
return torch.randn(*shape, device="cuda", dtype=dtype) * 0.2
def _rope_cache(max_position: int = 32) -> torch.Tensor:
inv_freq = 1.0 / (
50000 ** (torch.arange(0, 64, 2, dtype=torch.float32, device="cuda") / 64)
)
positions = torch.arange(max_position, dtype=torch.float32, device="cuda")
freqs = torch.outer(positions, inv_freq)
# The fused epilogue reads the cos/sin table in fp32 (RoPE math runs in fp32).
return torch.cat((freqs.cos(), freqs.sin()), dim=-1)
def _apply_gptj_rope(
x: torch.Tensor, positions: torch.Tensor, cos_sin_cache: torch.Tensor
) -> torch.Tensor:
cos, sin = cos_sin_cache.index_select(0, positions).chunk(2, dim=-1)
for _ in range(x.ndim - 2):
cos = cos.unsqueeze(1)
sin = sin.unsqueeze(1)
x1 = x[..., ::2].float()
x2 = x[..., 1::2].float()
out1 = x1 * cos.float() - x2 * sin.float()
out2 = x2 * cos.float() + x1 * sin.float()
return torch.stack((out1, out2), dim=-1).flatten(-2).to(x.dtype)
def _cache_rows(cache: torch.Tensor, slots: torch.Tensor) -> torch.Tensor:
return cache.reshape(-1, cache.shape[-1]).index_select(0, slots)
def _assert_fp8_close(actual: torch.Tensor, expected: torch.Tensor) -> None:
torch.testing.assert_close(
actual.float(),
expected.to(torch.float8_e4m3fn).float(),
atol=0.03125,
rtol=0.15,
)
def _strided_context_inputs(
num_tokens: int, num_heads: int, dtype: torch.dtype, k_pe_fp8: bool
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""The real prefill-context layouts: ``k_nope``/``v`` as the two strided
halves of one kv_b_proj output and ``k_pe`` as a column slice of the gather
workspace (left in the fp8 cache layout for a plain fp8 cache)."""
kv_nope = _randn(num_tokens, num_heads, 256, dtype=dtype)
k_nope, v = kv_nope.split((128, 128), dim=-1)
workspace = _randn(num_tokens, 576, dtype=dtype)
if k_pe_fp8:
workspace = workspace.to(torch.float8_e4m3fn)
return k_nope, workspace[:, 512:].unsqueeze(1), v
@pytest.mark.parametrize("num_tokens", [0, _NUM_TOKENS])
@pytest.mark.parametrize("num_heads", [3, _NUM_HEADS])
@pytest.mark.parametrize("input_dtype", [torch.bfloat16, torch.float16])
@torch.inference_mode()
def test_context_kv_concat_accepts_strided_inputs(
num_tokens: int, num_heads: int, input_dtype: torch.dtype
) -> None:
k_nope, k_pe, _ = _strided_context_inputs(
num_tokens, num_heads, input_dtype, k_pe_fp8=False
)
k = fused_mla_kv_concat(k_nope, k_pe)
assert k.is_contiguous()
assert k.dtype == input_dtype
torch.testing.assert_close(k[..., :128], k_nope, atol=0, rtol=0)
torch.testing.assert_close(
k[..., 128:], k_pe.expand(-1, num_heads, -1), atol=0, rtol=0
)
@pytest.mark.parametrize("k_pe_dtype", ["input", "fp8"])
@pytest.mark.parametrize("num_tokens", [0, _NUM_TOKENS])
@pytest.mark.parametrize("num_heads", [3, _NUM_HEADS])
@pytest.mark.parametrize("input_dtype", [torch.bfloat16, torch.float16])
@torch.inference_mode()
def test_context_kv_pack_quantizes_strided_inputs(
k_pe_dtype: str,
num_tokens: int,
num_heads: int,
input_dtype: torch.dtype,
) -> None:
k_nope, k_pe, v = _strided_context_inputs(
num_tokens, num_heads, input_dtype, k_pe_fp8=k_pe_dtype == "fp8"
)
k_actual, v_actual = fused_mla_kv_concat_quant_fp8(k_nope, k_pe, v)
fp8 = torch.float8_e4m3fn
k_expected = torch.empty_like(k_actual)
k_expected[..., :128] = k_nope.to(fp8)
k_expected[..., 128:] = k_pe.to(fp8)
assert k_actual.is_contiguous()
assert v_actual.is_contiguous()
# The pack casts with the native pairwise converters, so it must be
# bit-identical to torch's `.to(fp8)` rather than merely close.
torch.testing.assert_close(k_actual.float(), k_expected.float(), atol=0, rtol=0)
torch.testing.assert_close(v_actual.float(), v.to(fp8).float(), atol=0, rtol=0)
@pytest.mark.parametrize("cache_kind", ["bf16", "fp8", "fp8_ds_mla"])
@torch.inference_mode()
def test_prefill_epilogue_fuses_gptj_rope(cache_kind: str) -> None:
torch.manual_seed(0)
positions = torch.tensor(_POSITIONS, device="cuda", dtype=torch.int64)
slots = torch.tensor(_SLOTS, device="cuda", dtype=torch.int64)
cos_sin_cache = _rope_cache()
q = _randn(_NUM_TOKENS, _NUM_HEADS, 192)
k_nope = _randn(_NUM_TOKENS, _NUM_HEADS, 128)
k_pe = _randn(_NUM_TOKENS, 64)
kv_c = _randn(_NUM_TOKENS, 512)
v = _randn(_NUM_TOKENS, _NUM_HEADS, 128)
q_expected = q.clone()
q_expected[..., 128:] = _apply_gptj_rope(
q_expected[..., 128:], positions, cos_sin_cache
)
k_pe_expected = _apply_gptj_rope(k_pe, positions, cos_sin_cache)
k_expected = torch.cat(
(k_nope, k_pe_expected[:, None, :].expand(-1, _NUM_HEADS, -1)), dim=-1
)
cache_expected = torch.cat((kv_c, k_pe_expected), dim=-1)
if cache_kind == "bf16":
cache = torch.zeros(2, _BLOCK_SIZE, 576, device="cuda", dtype=_DTYPE)
q_actual = q.clone()
k_actual = fused_mla_key_concat_kv_cache_insert(
q_actual,
k_nope,
k_pe,
kv_c,
cache,
slots,
positions,
cos_sin_cache,
)
torch.testing.assert_close(q_actual, q_expected)
torch.testing.assert_close(k_actual, k_expected)
torch.testing.assert_close(_cache_rows(cache, slots), cache_expected)
elif cache_kind == "fp8":
cache = torch.zeros(
2, _BLOCK_SIZE, 576, device="cuda", dtype=torch.float8_e4m3fn
)
one = torch.ones(1, device="cuda", dtype=torch.float32)
q_actual, k_actual, v_actual = fused_mla_qkv_quant_kv_cache_fp8_insert(
q,
k_nope,
k_pe,
kv_c,
v,
cache,
slots,
one,
one,
one,
one,
positions,
cos_sin_cache,
)
_assert_fp8_close(q_actual, q_expected)
_assert_fp8_close(k_actual, k_expected)
_assert_fp8_close(v_actual, v)
_assert_fp8_close(_cache_rows(cache, slots), cache_expected)
else:
cache = torch.zeros(2, _BLOCK_SIZE, 656, device="cuda", dtype=torch.uint8)
q_actual = q.clone()
k_actual = fused_mla_key_concat_ds_mla_insert(
q_actual,
k_nope,
k_pe,
kv_c,
cache,
slots,
positions,
cos_sin_cache,
)
rope_cache = _cache_rows(cache, slots)[:, 528:656].view(_DTYPE)
torch.testing.assert_close(q_actual, q_expected)
torch.testing.assert_close(k_actual, k_expected)
torch.testing.assert_close(rope_cache, k_pe_expected)
@pytest.mark.parametrize("cache_kind", ["bf16", "fp8", "fp8_ds_mla"])
@torch.inference_mode()
def test_decode_epilogue_fuses_gptj_rope(cache_kind: str) -> None:
torch.manual_seed(1)
positions = torch.tensor(_POSITIONS, device="cuda", dtype=torch.int64)
slots = torch.tensor(_SLOTS, device="cuda", dtype=torch.int64)
cos_sin_cache = _rope_cache()
ql_nope = _randn(_NUM_TOKENS, _NUM_HEADS, 512)
q_pe = _randn(_NUM_TOKENS, _NUM_HEADS, 64)
kv_c = _randn(_NUM_TOKENS, 512)
k_pe = _randn(_NUM_TOKENS, 64)
q_pe_expected = _apply_gptj_rope(q_pe, positions, cos_sin_cache)
k_pe_expected = _apply_gptj_rope(k_pe, positions, cos_sin_cache)
q_expected = torch.cat((ql_nope, q_pe_expected), dim=-1)
cache_expected = torch.cat((kv_c, k_pe_expected), dim=-1)
kwargs = {"positions": positions, "cos_sin_cache": cos_sin_cache}
if cache_kind == "bf16":
cache = torch.zeros(2, _BLOCK_SIZE, 576, device="cuda", dtype=_DTYPE)
q_actual = fused_mla_decode_q_concat_kv_cache_insert(
ql_nope, q_pe, kv_c, k_pe, cache, slots, **kwargs
)
torch.testing.assert_close(q_actual, q_expected)
torch.testing.assert_close(_cache_rows(cache, slots), cache_expected)
elif cache_kind == "fp8":
cache = torch.zeros(
2, _BLOCK_SIZE, 576, device="cuda", dtype=torch.float8_e4m3fn
)
one = torch.ones(1, device="cuda", dtype=torch.float32)
q_actual = fused_mla_decode_q_concat_kv_cache_insert(
ql_nope,
q_pe,
kv_c,
k_pe,
cache,
slots,
q_scale_inv=one,
cache_scale_inv=one,
**kwargs,
)
_assert_fp8_close(q_actual, q_expected)
_assert_fp8_close(_cache_rows(cache, slots), cache_expected)
else:
cache = torch.zeros(2, _BLOCK_SIZE, 656, device="cuda", dtype=torch.uint8)
q_actual = fused_mla_decode_q_concat_kv_cache_insert(
ql_nope, q_pe, kv_c, k_pe, cache, slots, ds_mla=True, **kwargs
)
rope_cache = _cache_rows(cache, slots)[:, 528:656].view(_DTYPE)
torch.testing.assert_close(q_actual, q_expected)
torch.testing.assert_close(rope_cache, k_pe_expected)
@torch.inference_mode()
def test_decode_epilogue_preserves_nope_path() -> None:
torch.manual_seed(2)
slots = torch.tensor(_SLOTS, device="cuda", dtype=torch.int64)
ql_nope = _randn(_NUM_TOKENS, _NUM_HEADS, 512)
q_pe = _randn(_NUM_TOKENS, _NUM_HEADS, 64)
kv_c = _randn(_NUM_TOKENS, 512)
k_pe = _randn(_NUM_TOKENS, 64)
cache = torch.zeros(2, _BLOCK_SIZE, 576, device="cuda", dtype=_DTYPE)
q_actual = fused_mla_decode_q_concat_kv_cache_insert(
ql_nope, q_pe, kv_c, k_pe, cache, slots
)
torch.testing.assert_close(q_actual, torch.cat((ql_nope, q_pe), dim=-1))
torch.testing.assert_close(
_cache_rows(cache, slots), torch.cat((kv_c, k_pe), dim=-1)
)
@torch.inference_mode()
@pytest.mark.parametrize("num_tokens", [64, 65])
def test_decode_epilogue_row_split_boundary(num_tokens: int) -> None:
"""Both sides of the M-based warps-per-row dispatch produce the same rows."""
torch.manual_seed(3)
num_blocks = (num_tokens + _BLOCK_SIZE - 1) // _BLOCK_SIZE
slots = torch.randperm(num_blocks * _BLOCK_SIZE, device="cuda")[:num_tokens].long()
ql_nope = _randn(num_tokens, _NUM_HEADS, 512)
q_pe = _randn(num_tokens, _NUM_HEADS, 64)
kv_c = _randn(num_tokens, 512)
k_pe = _randn(num_tokens, 64)
cache = torch.zeros(num_blocks, _BLOCK_SIZE, 576, device="cuda", dtype=_DTYPE)
q_actual = fused_mla_decode_q_concat_kv_cache_insert(
ql_nope, q_pe, kv_c, k_pe, cache, slots
)
torch.testing.assert_close(q_actual, torch.cat((ql_nope, q_pe), dim=-1))
torch.testing.assert_close(
_cache_rows(cache, slots), torch.cat((kv_c, k_pe), dim=-1)
)
@@ -1,257 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
import vllm._custom_ops as ops
from vllm.models.kimi_k3.nvidia.ops.fused_mla_key_concat_kv_cache import (
fused_mla_decode_q_concat_kv_cache_insert,
fused_mla_key_concat_ds_mla_insert,
fused_mla_key_concat_kv_cache_insert,
fused_mla_qkv_quant_kv_cache_fp8_insert,
)
from vllm.platforms import current_platform
NUM_TOKENS = 4
NUM_HEADS = 2
KV_LORA_RANK = 512
NOPE_HEAD_DIM = 128
ROPE_HEAD_DIM = 64
QK_HEAD_DIM = NOPE_HEAD_DIM + ROPE_HEAD_DIM
V_HEAD_DIM = 128
CACHE_ENTRY = KV_LORA_RANK + ROPE_HEAD_DIM
DS_MLA_CACHE_ENTRY = 656
BLOCK_SIZE = 8
OWNED_TOKEN_INDICES = [1, 3]
pytestmark = pytest.mark.skipif(
not current_platform.is_cuda(), reason="Kimi-K3 fused MLA ops require CUDA"
)
def _inputs() -> dict[str, torch.Tensor]:
torch.manual_seed(0)
def make(*shape: int) -> torch.Tensor:
return torch.randn(*shape, device="cuda", dtype=torch.bfloat16)
return {
"q": make(NUM_TOKENS, NUM_HEADS, QK_HEAD_DIM),
"k_nope": make(NUM_TOKENS, NUM_HEADS, NOPE_HEAD_DIM),
"k_pe": make(NUM_TOKENS, ROPE_HEAD_DIM),
"kv_c": make(NUM_TOKENS, KV_LORA_RANK),
"v": make(NUM_TOKENS, NUM_HEADS, V_HEAD_DIM),
"ql_nope": make(NUM_TOKENS, NUM_HEADS, KV_LORA_RANK),
"q_pe": make(NUM_TOKENS, NUM_HEADS, ROPE_HEAD_DIM),
}
def _slot_mapping() -> torch.Tensor:
return torch.tensor([-1, 0, -1, 5], device="cuda", dtype=torch.int64)
def _owned(tensor: torch.Tensor) -> torch.Tensor:
return tensor[OWNED_TOKEN_INDICES]
def _full_key(k_nope: torch.Tensor, k_pe: torch.Tensor) -> torch.Tensor:
shared_pe = k_pe[:, None, :].expand(-1, k_nope.shape[1], -1)
return torch.cat((k_nope, shared_pe), dim=-1)
def _latent_query(ql_nope: torch.Tensor, q_pe: torch.Tensor) -> torch.Tensor:
return torch.cat((ql_nope, q_pe), dim=-1)
def _make_cache(cache_format: str) -> torch.Tensor:
if cache_format == "bf16":
return torch.full(
(1, BLOCK_SIZE, CACHE_ENTRY),
-7,
device="cuda",
dtype=torch.bfloat16,
)
if cache_format == "fp8":
return torch.full(
(1, BLOCK_SIZE, CACHE_ENTRY),
1.0,
device="cuda",
dtype=torch.float32,
).to(torch.float8_e4m3fn)
assert cache_format == "fp8_ds_mla"
return torch.full(
(1, BLOCK_SIZE, DS_MLA_CACHE_ENTRY),
165,
device="cuda",
dtype=torch.uint8,
)
def _reference_cache(
cache_format: str,
inputs: dict[str, torch.Tensor],
initial_cache: torch.Tensor,
slots: torch.Tensor,
) -> torch.Tensor:
reference_cache = initial_cache.clone()
valid_slots = _owned(slots)
if cache_format == "fp8_ds_mla":
ops.concat_and_cache_mla(
_owned(inputs["kv_c"]),
_owned(inputs["k_pe"]),
reference_cache,
valid_slots,
cache_format,
torch.ones(1, device="cuda", dtype=torch.float32),
)
return reference_cache
latent = torch.cat(
(_owned(inputs["kv_c"]), _owned(inputs["k_pe"])),
dim=-1,
)
flat_reference_cache = reference_cache.view(-1, CACHE_ENTRY)
for row, slot in zip(latent, valid_slots.tolist()):
flat_reference_cache[slot].copy_(row.to(reference_cache.dtype))
return reference_cache
def _assert_cache_matches_reference(
cache: torch.Tensor,
reference_cache: torch.Tensor,
initial_cache: torch.Tensor,
) -> None:
assert torch.equal(cache, reference_cache)
for slot in (0, 5):
assert not torch.equal(cache[0, slot], initial_cache[0, slot])
for slot in (1, 2, 3, 4, 6, 7):
assert torch.equal(cache[0, slot], initial_cache[0, slot])
@pytest.mark.parametrize("cache_format", ["bf16", "fp8_ds_mla"])
def test_prefill_concat_ignores_negative_slots_for_cache(cache_format: str) -> None:
inputs = _inputs()
slots = _slot_mapping()
initial_cache = _make_cache(cache_format)
mixed_cache = initial_cache.clone()
reference_cache = _reference_cache(cache_format, inputs, initial_cache, slots)
op = (
fused_mla_key_concat_kv_cache_insert
if cache_format == "bf16"
else fused_mla_key_concat_ds_mla_insert
)
output = op(
inputs["q"],
inputs["k_nope"],
inputs["k_pe"],
inputs["kv_c"],
mixed_cache,
slots,
)
torch.testing.assert_close(
output, _full_key(inputs["k_nope"], inputs["k_pe"]), rtol=0, atol=0
)
_assert_cache_matches_reference(mixed_cache, reference_cache, initial_cache)
def test_fp8_prefill_outputs_ignore_negative_slots_for_cache() -> None:
inputs = _inputs()
slots = _slot_mapping()
initial_cache = _make_cache("fp8")
mixed_cache = initial_cache.clone()
reference_cache = _reference_cache("fp8", inputs, initial_cache, slots)
scale_inv = torch.ones(1, device="cuda", dtype=torch.float32)
q, k, v = fused_mla_qkv_quant_kv_cache_fp8_insert(
inputs["q"],
inputs["k_nope"],
inputs["k_pe"],
inputs["kv_c"],
inputs["v"],
mixed_cache,
slots,
scale_inv,
scale_inv,
scale_inv,
scale_inv,
)
expected = (
inputs["q"].to(torch.float8_e4m3fn),
_full_key(inputs["k_nope"], inputs["k_pe"]).to(torch.float8_e4m3fn),
inputs["v"].to(torch.float8_e4m3fn),
)
for actual, reference in zip((q, k, v), expected):
torch.testing.assert_close(actual.float(), reference.float(), rtol=0, atol=0)
_assert_cache_matches_reference(mixed_cache, reference_cache, initial_cache)
@pytest.mark.parametrize("cache_format", ["bf16", "fp8", "fp8_ds_mla"])
def test_decode_concat_ignores_negative_slots_for_cache(cache_format: str) -> None:
inputs = _inputs()
slots = _slot_mapping()
initial_cache = _make_cache(cache_format)
mixed_cache = initial_cache.clone()
reference_cache = _reference_cache(cache_format, inputs, initial_cache, slots)
scale_inv = torch.ones(1, device="cuda", dtype=torch.float32)
output = fused_mla_decode_q_concat_kv_cache_insert(
inputs["ql_nope"],
inputs["q_pe"],
inputs["kv_c"],
inputs["k_pe"],
mixed_cache,
slots,
ds_mla=cache_format == "fp8_ds_mla",
q_scale_inv=scale_inv if cache_format == "fp8" else None,
cache_scale_inv=scale_inv if cache_format == "fp8" else None,
)
expected = _latent_query(inputs["ql_nope"], inputs["q_pe"])
if cache_format == "fp8":
expected = expected.to(torch.float8_e4m3fn)
torch.testing.assert_close(output.float(), expected.float(), rtol=0, atol=0)
else:
torch.testing.assert_close(output, expected, rtol=0, atol=0)
_assert_cache_matches_reference(mixed_cache, reference_cache, initial_cache)
@pytest.mark.parametrize("decode", [False, True])
def test_ds_mla_cache_insert_bit_compatible_with_reference(decode: bool) -> None:
"""Fused ds_mla insertion must match the reference bit-for-bit."""
torch.manual_seed(0)
num_tokens, num_blocks = 33, 16
dt = torch.bfloat16
kv_c = torch.randn(num_tokens, KV_LORA_RANK, device="cuda", dtype=dt)
k_pe = torch.randn(num_tokens, 1, ROPE_HEAD_DIM, device="cuda", dtype=dt)
k_nope = torch.randn(num_tokens, NUM_HEADS, NOPE_HEAD_DIM, device="cuda", dtype=dt)
q = torch.randn(num_tokens, NUM_HEADS, QK_HEAD_DIM, device="cuda", dtype=dt)
slots = torch.randperm(num_blocks * BLOCK_SIZE, device="cuda")[:num_tokens].long()
slots[torch.rand(num_tokens, device="cuda") < 0.3] = -1
ref = torch.full(
(num_blocks, BLOCK_SIZE, DS_MLA_CACHE_ENTRY),
165,
device="cuda",
dtype=torch.uint8,
)
got = ref.clone()
scale = torch.ones(1, device="cuda", dtype=torch.float32)
ops.concat_and_cache_mla(kv_c, k_pe.squeeze(1), ref, slots, "fp8_ds_mla", scale)
if decode:
ql_nope = torch.randn(
num_tokens, NUM_HEADS, KV_LORA_RANK, device="cuda", dtype=dt
)
fused_mla_decode_q_concat_kv_cache_insert(
ql_nope, q[..., -ROPE_HEAD_DIM:], kv_c, k_pe, got, slots, ds_mla=True
)
else:
fused_mla_key_concat_ds_mla_insert(q, k_nope, k_pe, kv_c, got, slots)
assert torch.equal(ref, got)
rows = got.view(-1, DS_MLA_CACHE_ENTRY)[slots[slots >= 0]]
scales = rows[:, KV_LORA_RANK : KV_LORA_RANK + 16].view(torch.float32)
torch.testing.assert_close(
torch.log2(scales), torch.log2(scales).round(), rtol=0, atol=0
)
File diff suppressed because it is too large Load Diff
@@ -1,143 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""fp8 query and index-K cache on the Triton indexer must agree with bf16.
The MiniMax M3 lightning-indexer scores only feed a top-k block ranking (the
attention itself reads the main KV cache), so an fp8 (e4m3) side cache trades
~2% relative score error for half the per-step index-K read bandwidth — the
dominant linear-in-context decode cost at long context. #45892 enabled fp8
index caches on the SM100 MSA path; this covers the Triton impl used
everywhere else (e.g. SM120), for both the decode and prefill scorers.
"""
import pytest
import torch
from vllm.models.minimax_m3.common.ops.index_topk import (
SPARSE_BLOCK_SIZE,
minimax_m3_index_decode,
minimax_m3_index_score,
minimax_m3_index_topk,
)
from vllm.platforms import current_platform
FP8_CUDA_SUPPORTED = current_platform.is_cuda() and current_platform.supports_fp8()
PAGE = SPARSE_BLOCK_SIZE
HEAD_DIM = 128
HEADS = 1
TOPK = 16
NUM_PAGES = 100
SEQ_LEN = 96 * PAGE + 37
PLANTED = (7, 40, 88)
def _make_inputs(device: str) -> tuple[torch.Tensor, torch.Tensor]:
torch.manual_seed(3)
keys = torch.randn(NUM_PAGES * PAGE, HEAD_DIM, dtype=torch.bfloat16, device=device)
# RMS-normalize like the real (post index_k_norm) keys.
keys = keys * keys.float().pow(2).mean(-1, keepdim=True).add(1e-6).rsqrt().to(
torch.bfloat16
)
q = torch.randn(1, HEADS, HEAD_DIM, dtype=torch.bfloat16, device=device)
for b in PLANTED:
keys[b * PAGE + 5] = (q[0, 0] * 3).to(torch.bfloat16)
return q, keys
def _check_selection(top_bf16: set[int], top_fp8: set[int]) -> None:
# Planted (clearly relevant) blocks must never be lost.
assert all(b in top_fp8 for b in PLANTED)
# Selection may differ only at the noise floor (near-tied filler blocks).
assert len(top_bf16 & top_fp8) >= TOPK - 2
@pytest.mark.skipif(not FP8_CUDA_SUPPORTED, reason="CUDA FP8 support required")
@torch.inference_mode()
def test_fp8_index_query_and_cache_decode_topk_matches_bf16() -> None:
dev = "cuda"
q, keys = _make_inputs(dev)
block_table = torch.arange(NUM_PAGES, dtype=torch.int32, device=dev).unsqueeze(0)
seq_lens = torch.tensor([SEQ_LEN], dtype=torch.int32, device=dev)
def select(cache: torch.Tensor) -> set[int]:
query = q.to(cache.dtype)
idx = minimax_m3_index_decode(
query,
cache.view(NUM_PAGES, PAGE, HEAD_DIM).contiguous(),
block_table,
seq_lens,
SEQ_LEN,
TOPK,
0,
0,
HEADS,
1,
1,
)
return {int(b) for b in idx[0, 0].tolist() if b >= 0}
_check_selection(select(keys), select(keys.to(torch.float8_e4m3fn)))
@pytest.mark.skipif(not FP8_CUDA_SUPPORTED, reason="CUDA FP8 support required")
@torch.inference_mode()
def test_fp8_index_query_and_cache_prefill_topk_matches_bf16() -> None:
dev = "cuda"
q, keys = _make_inputs(dev)
block_table = torch.arange(NUM_PAGES, dtype=torch.int32, device=dev).unsqueeze(0)
seq_lens = torch.tensor([SEQ_LEN], dtype=torch.int32, device=dev)
# single decode-like query token at the end of the sequence
cu_seqlens_q = torch.tensor([0, 1], dtype=torch.int32, device=dev)
prefix_lens = torch.tensor([SEQ_LEN - 1], dtype=torch.int32, device=dev)
def select(cache: torch.Tensor) -> set[int]:
query = q.to(cache.dtype)
score = minimax_m3_index_score(
query,
cache.view(NUM_PAGES, PAGE, HEAD_DIM).contiguous(),
block_table,
cu_seqlens_q,
seq_lens,
prefix_lens,
1,
SEQ_LEN,
HEADS,
)
idx = minimax_m3_index_topk(score, cu_seqlens_q, prefix_lens, 1, TOPK, 0, 0)
return {int(b) for b in idx[0, 0].tolist() if b >= 0}
_check_selection(select(keys), select(keys.to(torch.float8_e4m3fn)))
@pytest.mark.skipif(not FP8_CUDA_SUPPORTED, reason="CUDA FP8 support required")
@torch.inference_mode()
def test_bf16_query_against_fp8_cache_decode_topk_matches_bf16() -> None:
"""A mixed pair must upcast rather than emit a mixed-dtype fp8 dot.
Production allocates index_q with the index-K cache dtype, so the scorers
normally see a matched pair and keep the native FP8 MMA. This pins the
other branch: a bf16 query against an e4m3 cache still compiles and still
ranks the planted blocks.
"""
dev = "cuda"
q, keys = _make_inputs(dev)
block_table = torch.arange(NUM_PAGES, dtype=torch.int32, device=dev).unsqueeze(0)
seq_lens = torch.tensor([SEQ_LEN], dtype=torch.int32, device=dev)
def select(cache: torch.Tensor) -> set[int]:
idx = minimax_m3_index_decode(
q, # bf16 query, deliberately not cast to the cache dtype
cache.view(NUM_PAGES, PAGE, HEAD_DIM).contiguous(),
block_table,
seq_lens,
SEQ_LEN,
TOPK,
0,
0,
HEADS,
1,
1,
)
return {int(b) for b in idx[0, 0].tolist() if b >= 0}
_check_selection(select(keys), select(keys.to(torch.float8_e4m3fn)))
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -1,226 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""ROCm correctness tests for the strided DSA candidate mask.
``_apply_candidate_mask_strided`` is a decode-path variant of the shared
``apply_candidate_mask``. It differs in two ways that need pinning down:
* it launches a fixed number of programs that stride the column axis, rather
than one program per 1024-column tile, so at ``max_model_len`` widths the
grid no longer scales with a workspace that is mostly padding;
* it stops at each row's ``end`` instead of sanitizing the whole width.
The second is only sound because every consumer bounds its scan by the same
ends -- ``top_k_per_row_decode`` takes ``rowEnd`` from ``seq_lens``. So the
contract is agreement with the shared kernel over ``[0, end)`` and nothing
beyond it, which is what these tests assert rather than comparing whole rows.
"""
import pytest
import torch
from vllm.platforms import current_platform
pytestmark = pytest.mark.skipif(
not current_platform.is_rocm(), reason="ROCm-specific tests"
)
BLOCK_SIZE = 8
TOPK_BLOCKS = 64
# Above 128 tiles the grid saturates and programs start striding; below it the
# min() clamp applies instead. Both regimes need covering, so the widths here
# deliberately straddle 128 * 1024.
NARROW = 8192
WIDE = 262144
def _inputs(rows, width, ends, starts, *, block_size, col_stride, dtype):
"""Logits, bounds and candidate blocks shared by both kernels."""
torch.manual_seed(0)
# A column stride > 1 keeps the kernels honest about stride_col; the decode
# caller slices a workspace, so contiguous rows are not guaranteed.
logits = torch.randn(rows, width * col_stride, device="cuda")[:, ::col_stride]
logits[0, : min(16, width)] = 0.0
if rows > 2:
logits[2, 5] = float("nan")
row_ke = torch.tensor(ends, device="cuda", dtype=dtype)
row_ks = torch.tensor(starts, device="cuda", dtype=dtype) if starts else None
nblocks = (width + block_size - 1) // block_size
candidates = torch.randint(
0, nblocks, (rows, TOPK_BLOCKS), device="cuda", dtype=torch.int32
)
# -1 is the "no block" sentinel the selector emits for short rows.
candidates[:, ::7] = -1
return logits, row_ks, row_ke, candidates
def _run_both(logits, row_ks, row_ke, candidates, block_size, row_repeat):
from vllm.model_executor.kernels.attention.dsa.candidate_blocks import (
apply_candidate_mask,
)
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
_apply_candidate_mask_strided,
)
ref = logits.clone()
got = logits.clone()
apply_candidate_mask(ref, row_ks, row_ke, candidates, block_size, row_repeat)
_apply_candidate_mask_strided(
got, row_ks, row_ke, candidates, block_size, row_repeat
)
return ref, got
def _assert_agrees_within_ends(ref, got, row_ke, row_repeat):
"""Compare only the region a consumer can read."""
for row in range(ref.shape[0]):
end = int(row_ke[row // row_repeat])
if end <= 0:
continue
torch.testing.assert_close(
got[row, :end],
ref[row, :end],
rtol=0,
atol=0,
equal_nan=True,
msg=lambda m, r=row, e=end: f"row {r} differs within [0, {e}): {m}",
)
@pytest.mark.parametrize("width", [NARROW, WIDE])
@pytest.mark.parametrize("col_stride", [1, 2])
def test_matches_shared_kernel_ragged_ends(width, col_stride):
"""Ends on and around tile boundaries, plus the empty and full rows."""
ends = [0, 1, 1023, 1024, 1025, width - 1, width, width // 2]
logits, row_ks, row_ke, candidates = _inputs(
len(ends),
width,
ends,
None,
block_size=BLOCK_SIZE,
col_stride=col_stride,
dtype=torch.int32,
)
ref, got = _run_both(logits, row_ks, row_ke, candidates, BLOCK_SIZE, 1)
_assert_agrees_within_ends(ref, got, row_ke, 1)
@pytest.mark.parametrize("block_size", [8, 16])
def test_matches_shared_kernel_with_starts(block_size):
"""Non-zero starts: the decode caller passes zeros, prefill does not."""
ends = [0, 64, 4096, 8192]
starts = [0, 7, 1024, 3]
logits, row_ks, row_ke, candidates = _inputs(
len(ends),
NARROW,
ends,
starts,
block_size=block_size,
col_stride=1,
dtype=torch.int64,
)
ref, got = _run_both(logits, row_ks, row_ke, candidates, block_size, 1)
_assert_agrees_within_ends(ref, got, row_ke, 1)
def test_matches_shared_kernel_row_repeat():
"""Speculative decode: next_n logit rows share one bound."""
row_repeat = 3
ends = [0, 1025, WIDE]
logits, row_ks, row_ke, candidates = _inputs(
len(ends) * row_repeat,
WIDE,
ends,
None,
block_size=BLOCK_SIZE,
col_stride=1,
dtype=torch.int32,
)
ref, got = _run_both(logits, row_ks, row_ke, candidates, BLOCK_SIZE, row_repeat)
_assert_agrees_within_ends(ref, got, row_ke, row_repeat)
def test_leaves_columns_past_end_untouched():
"""The deliberate difference from the shared kernel, pinned explicitly.
Skipping the tail is where the speedup comes from, so a change that
quietly restores full-width sanitizing should fail here rather than just
get slower. Work stops at the tile boundary containing ``end``, not at
``end`` itself, so that is the bound asserted.
"""
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
_MASK_TILE,
_apply_candidate_mask_strided,
)
ends = [0, 1, 1025, WIDE // 2]
logits, row_ks, row_ke, candidates = _inputs(
len(ends),
WIDE,
ends,
None,
block_size=BLOCK_SIZE,
col_stride=1,
dtype=torch.int32,
)
sentinel = 1.5
logits.fill_(sentinel)
_apply_candidate_mask_strided(logits, row_ks, row_ke, candidates, BLOCK_SIZE, 1)
for row, end in enumerate(ends):
touched = min(-(-end // _MASK_TILE) * _MASK_TILE, WIDE)
tail = logits[row, touched:]
assert torch.equal(tail, torch.full_like(tail, sentinel)), (
f"row {row} (end {end}) was written past column {touched}"
)
# And the bound is tight: a row that ends early must not have cost a
# full-width pass, which is the whole point of the kernel.
assert touched <= end + _MASK_TILE
def test_cudagraph_replay_tracks_changing_ends():
"""The grid is static so a FULL capture stays valid when ends change.
Capturing bakes the grid, so a data-dependent program count would freeze
whatever lengths were resident at capture. Replaying against longer and
shorter contexts than the captured ones is the property that matters.
"""
from vllm.model_executor.kernels.attention.dsa.candidate_blocks import (
apply_candidate_mask,
)
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
_apply_candidate_mask_strided,
)
rows = 4
captured_ends = [1024, 4096, 8192, 2048]
logits, row_ks, row_ke, candidates = _inputs(
rows,
WIDE,
captured_ends,
None,
block_size=BLOCK_SIZE,
col_stride=1,
dtype=torch.int32,
)
base = logits.clone()
# Warm up outside the capture: Triton compiles on first call, and autotune
# launches would otherwise happen inside the graph.
_apply_candidate_mask_strided(
logits.clone(), row_ks, row_ke, candidates, BLOCK_SIZE, 1
)
torch.accelerator.synchronize()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
_apply_candidate_mask_strided(logits, row_ks, row_ke, candidates, BLOCK_SIZE, 1)
for replay_ends in ([1, 131072, WIDE, 0], [WIDE, 1025, 3, 65536]):
row_ke.copy_(torch.tensor(replay_ends, device="cuda", dtype=row_ke.dtype))
logits.copy_(base)
graph.replay()
torch.accelerator.synchronize()
ref = base.clone()
apply_candidate_mask(ref, row_ks, row_ke, candidates, BLOCK_SIZE, 1)
_assert_agrees_within_ends(ref, logits, row_ke, 1)
File diff suppressed because it is too large Load Diff
@@ -1,262 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Regression test for causal masking in the AITER MLA verify MTP path.
When ``num_heads < 16`` the ROCm AITER MLA backend cannot use the ASM decode
kernel for a multi-token block, so ``AiterMLAImpl.forward_mqa`` routes verify
through ``mla_gluon``'s native 4-D MTP entry (``q`` shaped
``[batch, qlen, nhead, dim]`` with ``use_2d_view=False``). aiter applies the
causal tail internally: position ``t`` attends KV ``[0, seq_len - qlen + t]``.
The test drives the real builder and ``forward_mqa`` over a multi-token decode
batch, intercepts the arguments actually handed to the Gluon kernel, and pins
that the 4-D MTP contract is used with per-request ``block_table`` and
``cache_seqlens`` rather than a flattened per-token paged-KV view.
"""
import types
from unittest.mock import patch
import pytest
import torch
from vllm._aiter_ops import is_aiter_found
from vllm.platforms import current_platform
def _on_rocm_with_aiter() -> bool:
return current_platform.is_rocm() and is_aiter_found()
# The production verify MTP path is gfx950-only. This test replaces the Gluon
# kernel with a spy and forces the architecture feature probe so the metadata
# regression remains testable on every ROCm AITER CI runner.
pytestmark = pytest.mark.skipif(
not _on_rocm_with_aiter(),
reason="ROCM_AITER_MLA verify MTP requires ROCm and AITER",
)
# Below the 16-head threshold that selects the MTP path. DeepSeek-V3 / R1 at
# TP=16 and Kimi-K3 at TP=8 both land here.
NUM_QUERY_HEADS = 12
KV_LORA_RANK = 512
QK_NOPE_HEAD_DIM = 128
QK_ROPE_HEAD_DIM = 64
V_HEAD_DIM = 128
HEAD_SIZE = KV_LORA_RANK + QK_ROPE_HEAD_DIM
# 1 + num_speculative_tokens, i.e. the verify block length.
QLEN = 8
# Committed context per request, including a fresh request at zero context
# where the whole KV range is the verify block itself.
CONTEXT_LENS = [0, 1, 37, 512]
# A cudagraph padding request: seq_len 0.
PADDING_ROWS = 1
# The Gluon path flattens the KV cache to one page per token.
PAGE_SIZE = 1
MAX_MODEL_LEN = 1024
MAX_NUM_SEQS = 8
# The builder resolves the verify routing and records it on the metadata, so
# the gfx950 feature probe has to be forced for the build as well as for the
# forward; ROCm CI also runs gfx942, where the real probe is False.
_GLUON_SUPPORTED_TARGET = (
"vllm.v1.attention.backends.mla.rocm_aiter_mla._gluon_mla_decode_supported"
)
def _seq_lens() -> list[int]:
return [c + QLEN for c in CONTEXT_LENS] + [0] * PADDING_ROWS
def _run_verify_block():
"""Drive the real builder + forward_mqa over one multi-token verify block.
Returns ``(metadata, captured)`` where ``captured`` holds the Gluon kwargs.
"""
from tests.v1.attention.utils import (
BatchSpec,
create_common_attn_metadata,
create_vllm_config,
)
from vllm.config import SpeculativeConfig
from vllm.config.vllm import set_current_vllm_config
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAImpl
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.kv_cache_interface import MLAAttentionSpec
from vllm.v1.worker.workspace import init_workspace_manager
device = torch.device("cuda:0")
seq_lens = _seq_lens()
num_reqs = len(seq_lens)
# One flat page per token, arange block indices, plus room to spare.
num_gpu_blocks = num_reqs * max(seq_lens) + 256
vllm_config = create_vllm_config(
model_name="deepseek-ai/DeepSeek-R1",
max_model_len=MAX_MODEL_LEN,
num_gpu_blocks=num_gpu_blocks,
block_size=PAGE_SIZE,
max_num_seqs=MAX_NUM_SEQS,
max_num_batched_tokens=8192,
hf_config_override={"num_attention_heads": NUM_QUERY_HEADS},
)
# What raises reorder_batch_threshold to QLEN, so a QLEN-token block is
# classified as a decode instead of a prefill. ngram needs no draft model.
vllm_config.speculative_config = SpeculativeConfig(
method="ngram", num_speculative_tokens=QLEN - 1
)
vllm_config.model_config.get_num_attention_heads = types.MethodType( # type: ignore[method-assign]
lambda self, parallel_config, arch_config=None: NUM_QUERY_HEADS,
vllm_config.model_config,
)
spec = MLAAttentionSpec(
block_size=PAGE_SIZE,
num_kv_heads=1,
head_size=vllm_config.model_config.get_head_size(),
dtype=vllm_config.model_config.dtype,
cache_dtype_str="auto",
)
backend_cls = AttentionBackendEnum.ROCM_AITER_MLA.get_class()
builder_cls = backend_cls.get_builder_cls()
impl_cls = backend_cls.get_impl_cls()
# The builder reads layer.prefill_backend from static_forward_context; a
# stub with the attribute is enough for metadata construction.
layer_name = "placeholder"
vllm_config.compilation_config.static_forward_context[layer_name] = (
types.SimpleNamespace(
prefill_backend=torch.empty((1,)),
q_lora_rank=None,
kv_lora_rank=KV_LORA_RANK,
qk_nope_head_dim=QK_NOPE_HEAD_DIM,
qk_rope_head_dim=QK_ROPE_HEAD_DIM,
v_head_dim=V_HEAD_DIM,
)
)
init_workspace_manager(device)
batch_spec = BatchSpec(seq_lens=seq_lens, query_lens=[QLEN] * num_reqs)
captured: dict = {}
def spy(**kwargs):
captured.update(kwargs)
captured["q_nope"] = kwargs["q_nope"].detach().clone()
captured["page_table"] = kwargs["page_table"].detach().clone()
captured["seq_info"] = kwargs["seq_info"].detach().clone()
with (
patch(_GLUON_SUPPORTED_TARGET, lambda: True),
set_current_vllm_config(vllm_config),
):
builder = builder_cls(spec, [layer_name], vllm_config, device)
common_attn_metadata = create_common_attn_metadata(
batch_spec, PAGE_SIZE, device, arange_block_indices=True
)
metadata = builder.build(
common_prefix_len=0, common_attn_metadata=common_attn_metadata
)
impl = impl_cls(
num_heads=NUM_QUERY_HEADS,
head_size=HEAD_SIZE,
scale=(QK_NOPE_HEAD_DIM + QK_ROPE_HEAD_DIM) ** -0.5,
num_kv_heads=1,
alibi_slopes=None,
sliding_window=None,
kv_cache_dtype="auto",
logits_soft_cap=None,
attn_type="decoder",
kv_sharing_target_layer_name=None,
q_lora_rank=None,
kv_lora_rank=KV_LORA_RANK,
qk_nope_head_dim=QK_NOPE_HEAD_DIM,
qk_rope_head_dim=QK_ROPE_HEAD_DIM,
qk_head_dim=QK_NOPE_HEAD_DIM + QK_ROPE_HEAD_DIM,
v_head_dim=V_HEAD_DIM,
kv_b_proj=None,
)
num_rows = num_reqs * QLEN
dtype = torch.bfloat16
q_nope = torch.zeros(
num_rows, NUM_QUERY_HEADS, KV_LORA_RANK, dtype=dtype, device=device
)
q_pe = torch.zeros(
num_rows, NUM_QUERY_HEADS, QK_ROPE_HEAD_DIM, dtype=dtype, device=device
)
kv_cache = torch.zeros(
num_gpu_blocks, PAGE_SIZE, HEAD_SIZE, dtype=dtype, device=device
)
# The Gluon kernel only reads the metadata this test is about, so a spy in
# its place keeps the assertions independent of the AITER build.
with (
patch(_GLUON_SUPPORTED_TARGET, lambda: True),
patch(
"vllm.v1.attention.backends.mla.rocm_aiter_mla._get_mla_gluon",
lambda: spy,
),
):
assert isinstance(impl, AiterMLAImpl)
impl.forward_mqa((q_nope, q_pe), kv_cache, metadata, layer=None)
return metadata, captured
def test_verify_mtp_uses_native_4d_gluon_entry():
"""Verify uses 4-D q with per-request paged_kv metadata.
Regression guard: before the fix every flattened row of a request got that
request's whole paged-KV range, so verify position t attended to the draft
tokens after it. The native MTP path delegates causal masking to aiter.
"""
metadata, captured = _run_verify_block()
decode = metadata.decode
assert decode is not None, "batch was not classified as a decode"
assert decode.max_qo_len == QLEN, (
f"expected a {QLEN}-token verify block, got max_qo_len="
f"{decode.max_qo_len}; the MTP path under test was not reached"
)
assert decode.use_gluon_verify, (
"the builder owns verify routing, so the metadata -- not the impl -- "
"has to select the Gluon MTP entry for a small-head bf16 verify block"
)
assert captured, "forward_mqa did not reach the Gluon kernel"
seq_lens = _seq_lens()
num_reqs = len(seq_lens)
q_nope = captured["q_nope"]
assert q_nope.dim() == 4, (
f"verify must pass 4-D q to mla_gluon, got {q_nope.dim()}-D shape "
f"{tuple(q_nope.shape)}"
)
assert q_nope.shape[:2] == (num_reqs, QLEN), (
f"expected q shape [num_reqs={num_reqs}, qlen={QLEN}, ...], "
f"got {tuple(q_nope.shape)}"
)
assert q_nope.shape[2] == NUM_QUERY_HEADS
assert captured["use_2d_view"] is False
page_table = captured["page_table"]
seq_info = captured["seq_info"]
assert page_table.dim() == 1, (
f"verify MTP passes the 1-D paged_kv_indices buffer, got {page_table.dim()}-D"
)
assert seq_info.dim() == 1 and seq_info.numel() == num_reqs + 1, (
f"verify MTP passes paged_kv_indptr [num_reqs + 1], got shape "
f"{tuple(seq_info.shape)}"
)
assert decode is not None and decode.paged_kv_indptr is not None
assert torch.equal(seq_info, decode.paged_kv_indptr[: num_reqs + 1])
assert torch.equal(
page_table[: int(seq_info[-1].item())],
decode.paged_kv_indices[: int(seq_info[-1].item())],
)
@@ -1,225 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""ROCm kernel correctness tests for AITER MLA decode.
Compares ``rocm_aiter_ops.mla_decode_fwd`` against a pure PyTorch reference
under the absorbed MLA formulation (DeepSeek-V3/V4) with varied batch sizes,
head counts, and sequence lengths.
"""
import pytest
import torch
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
_SKIP_NON_MI3XX = True
if current_platform.is_rocm():
from vllm.platforms.rocm import on_mi3xx
_SKIP_NON_MI3XX = not on_mi3xx()
pytestmark = [
pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-specific tests"),
pytest.mark.skipif(_SKIP_NON_MI3XX, reason="MI300/MI350 ROCm only"),
]
# DeepSeek-V3/V4 MLA config.
Q_HEAD_DIM = 576 # kv_lora_rank + qk_rope_head_dim
V_HEAD_DIM = 512 # kv_lora_rank
SM_SCALE = Q_HEAD_DIM**-0.5
NUM_HEADS = [16, 128]
BATCH_SIZES = [1, 4, 16]
KV_SEQ_LENS = [16, 256]
ATOL, RTOL = 1.5e-2, 1e-2
def _require_aiter():
from vllm._aiter_ops import is_aiter_found_and_supported
if not is_aiter_found_and_supported():
pytest.skip("aiter is required on supported ROCm hardware for this test")
def _ref_mla_decode(
q: torch.Tensor,
kv_buffer: torch.Tensor,
kv_indptr: torch.Tensor,
kv_indices: torch.Tensor,
) -> torch.Tensor:
"""Pure PyTorch reference for absorbed MLA decode.
K uses full q_head_dim (576) for scoring, V uses first v_head_dim (512)
for output. This mirrors the production kernel's absorbed formulation.
"""
batch_size, num_heads, _ = q.shape
output = torch.zeros(
batch_size, num_heads, V_HEAD_DIM, dtype=q.dtype, device=q.device
)
for b in range(batch_size):
start = kv_indptr[b].item()
end = kv_indptr[b + 1].item()
token_indices = kv_indices[start:end]
k = kv_buffer[token_indices].float()
v = kv_buffer[token_indices, :V_HEAD_DIM].float()
for h in range(num_heads):
q_h = q[b, h, :].float()
scores = torch.mv(k, q_h) * SM_SCALE
weights = torch.softmax(scores, dim=0)
output[b, h, :] = torch.mv(v.t(), weights).to(q.dtype)
return output
def _make_inputs(
batch_size: int,
nhead: int,
kv_seq_len: int,
*,
contiguous_indices: bool = True,
):
"""Build valid MLA decode inputs on the current CUDA device.
When contiguous_indices=False, tokens are randomly scattered in a 2x pool
to simulate real paged allocation.
"""
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAHelper
actual_nhead = AiterMLAHelper.get_actual_mla_num_heads(nhead)
total_kv_tokens = batch_size * kv_seq_len
pool_size = total_kv_tokens if contiguous_indices else total_kv_tokens * 2
q = torch.randn(
batch_size, actual_nhead, Q_HEAD_DIM, dtype=torch.bfloat16, device="cuda"
)
kv_buffer = torch.randn(
pool_size, 1, Q_HEAD_DIM, dtype=torch.bfloat16, device="cuda"
)
o = torch.zeros(
batch_size, actual_nhead, V_HEAD_DIM, dtype=torch.bfloat16, device="cuda"
)
qo_indptr = torch.arange(0, batch_size + 1, dtype=torch.int32, device="cuda")
kv_indptr = (
torch.arange(0, batch_size + 1, dtype=torch.int32, device="cuda") * kv_seq_len
)
kv_last_page_lens = torch.ones(batch_size, dtype=torch.int32, device="cuda")
if contiguous_indices:
kv_indices = torch.arange(0, total_kv_tokens, dtype=torch.int32, device="cuda")
else:
perm = torch.randperm(pool_size, device="cuda")[:total_kv_tokens]
kv_indices = perm.to(torch.int32)
return {
"q": q,
"kv_buffer": kv_buffer,
"o": o,
"qo_indptr": qo_indptr,
"kv_indptr": kv_indptr,
"kv_indices": kv_indices,
"kv_last_page_lens": kv_last_page_lens,
"actual_nhead": actual_nhead,
}
def _run_kernel(inputs: dict) -> torch.Tensor:
"""Run decode through the production rocm_aiter_ops path."""
from vllm._aiter_ops import rocm_aiter_ops
rocm_aiter_ops.mla_decode_fwd(
inputs["q"],
inputs["kv_buffer"],
inputs["o"],
SM_SCALE,
inputs["qo_indptr"],
1,
inputs["kv_indptr"],
inputs["kv_indices"],
inputs["kv_last_page_lens"],
)
return inputs["o"]
def _ref_output(inputs: dict) -> torch.Tensor:
"""Compute reference output from the same inputs."""
kv_flat = inputs["kv_buffer"].squeeze(1)
return _ref_mla_decode(
inputs["q"], kv_flat, inputs["kv_indptr"], inputs["kv_indices"]
)
@pytest.mark.parametrize("nhead", NUM_HEADS)
@pytest.mark.parametrize("batch_size", BATCH_SIZES)
@pytest.mark.parametrize("kv_seq_len", KV_SEQ_LENS)
@torch.inference_mode()
def test_mla_decode_accuracy(
nhead: int,
batch_size: int,
kv_seq_len: int,
) -> None:
"""BF16 decode accuracy vs PyTorch reference (contiguous indices)."""
_require_aiter()
set_random_seed(0)
inputs = _make_inputs(batch_size, nhead, kv_seq_len)
output = _run_kernel(inputs)
output_ref = _ref_output(inputs)
torch.testing.assert_close(output, output_ref, atol=ATOL, rtol=RTOL)
@pytest.mark.parametrize("nhead", NUM_HEADS)
@pytest.mark.parametrize("batch_size", [1, 4, 8])
@pytest.mark.parametrize("kv_seq_len", [16, 128])
@torch.inference_mode()
def test_mla_decode_noncontiguous_indices(
nhead: int,
batch_size: int,
kv_seq_len: int,
) -> None:
"""BF16 decode accuracy with shuffled kv_indices (paged allocation)."""
_require_aiter()
set_random_seed(0)
inputs = _make_inputs(batch_size, nhead, kv_seq_len, contiguous_indices=False)
output = _run_kernel(inputs)
output_ref = _ref_output(inputs)
torch.testing.assert_close(output, output_ref, atol=ATOL, rtol=RTOL)
@torch.inference_mode()
def test_mla_decode_determinism() -> None:
"""Repeated decode calls produce bitwise-identical results."""
_require_aiter()
set_random_seed(0)
inputs = _make_inputs(batch_size=4, nhead=128, kv_seq_len=16)
out_first = _run_kernel(inputs).clone()
for _ in range(3):
inputs["o"] = torch.zeros_like(inputs["o"])
out = _run_kernel(inputs)
torch.testing.assert_close(out, out_first, atol=0, rtol=0)
@torch.inference_mode()
def test_mla_decode_smoke() -> None:
"""Basic sanity: correct shape, dtype, finite, non-zero."""
_require_aiter()
set_random_seed(0)
inputs = _make_inputs(batch_size=4, nhead=128, kv_seq_len=64)
output = _run_kernel(inputs)
assert output.shape == (4, 128, V_HEAD_DIM)
assert output.dtype == torch.bfloat16
assert torch.isfinite(output).all()
assert not torch.all(output == 0)
@@ -1,213 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Regression test for AITER MLA persistent decode metadata dtypes.
For the gfx950 fp8/fp8 nhead=32 qlen=1 fold path, the split/reduce metadata
layout depends on the q/kv element size. The builder must forward dtype_q/dtype_kv
to ``get_mla_metadata_v1``; omitting them lays out the work for the wrong dtype
and corrupts decode output. The test pins the builder's metadata to a golden
recomputed at runtime with the explicit correct dtypes.
"""
import types
from unittest.mock import patch
import pytest
import torch
from vllm._aiter_ops import is_aiter_found
from vllm.platforms import current_platform
def _on_gfx950() -> bool:
if not (current_platform.is_rocm() and is_aiter_found()):
return False
from vllm.platforms.rocm import on_gfx950
return on_gfx950()
pytestmark = pytest.mark.skipif(
not _on_gfx950(),
reason="AITER MLA fp8 persistent decode metadata is gfx950-only",
)
# The fold path that the bug corrupted: fp8 query + fp8 KV-cache, 32 query
# heads, single-token decode, batch 128, context 8192, page_size 1.
NUM_QUERY_HEADS = 32
DECODE_QLEN = 1
BATCH_SIZE = 128
CONTEXT_LEN = 8192
PAGE_SIZE = 1
# On the fp8 KV-cache path the builder forwards the platform fp8 dtype (aiter
# dtypes.fp8) for both q and kv. Mirror it via current_platform.fp8_dtype()
# instead of hardcoding a literal (see #47276).
EXPECTED_Q_DTYPE = current_platform.fp8_dtype()
EXPECTED_KV_DTYPE = current_platform.fp8_dtype()
# The split/reduce content tensors filled by get_mla_metadata_v1. work_meta_data
# is excluded: it holds raw device pointers, never equal across allocations.
_CONTENT_METADATA_FIELDS = (
"work_indptr",
"work_info_set",
"reduce_indptr",
"reduce_final_map",
"reduce_partial_map",
)
# The builder's get_mla_metadata_v1 call passes 6 input args then 6 output
# buffers (see AiterMLAMetadataBuilder._build_decode). Output order -> field.
_NUM_INPUT_ARGS = 6
_OUTPUT_ARG_FIELDS = (
"work_meta_data",
"work_info_set",
"work_indptr",
"reduce_indptr",
"reduce_final_map",
"reduce_partial_map",
)
def _build_decode_metadata():
"""Build AITER MLA decode metadata for the fp8/fp8 nhead=32 fold path.
Returns ``(metadata, captured)`` where ``captured`` records the positional
args/kwargs the builder passed to ``get_mla_metadata_v1``, so the golden can
be recomputed from the identical inputs.
"""
from tests.v1.attention.utils import (
BatchSpec,
create_common_attn_metadata,
create_vllm_config,
)
from vllm.config.vllm import set_current_vllm_config
from vllm.model_executor.layers.attention.mla_attention import get_mla_dims
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.kv_cache_interface import MLAAttentionSpec
from vllm.v1.worker.workspace import init_workspace_manager
device = torch.device("cuda:0")
vllm_config = create_vllm_config(
model_name="deepseek-ai/DeepSeek-R1",
max_model_len=CONTEXT_LEN,
# One flat page per token (page_size=1); +buffer for the null block.
num_gpu_blocks=BATCH_SIZE * CONTEXT_LEN + 200,
block_size=PAGE_SIZE,
max_num_seqs=BATCH_SIZE,
max_num_batched_tokens=8192,
hf_config_override={"num_attention_heads": NUM_QUERY_HEADS},
)
vllm_config.cache_config.cache_dtype = "fp8"
spec = MLAAttentionSpec(
block_size=PAGE_SIZE,
num_kv_heads=1,
head_size=vllm_config.model_config.get_head_size(),
dtype=vllm_config.model_config.dtype,
cache_dtype_str="fp8",
)
builder_cls = AttentionBackendEnum.ROCM_AITER_MLA.get_class().get_builder_cls()
# The builder reads prefill_backend and the MLA latent dims off the layer,
# so the stub carries both; the dims come from the model config to stay in
# step with the checkpoint.
mla_dims = get_mla_dims(vllm_config.model_config)
layer_name = "placeholder"
vllm_config.compilation_config.static_forward_context[layer_name] = (
types.SimpleNamespace(
prefill_backend=torch.empty((1,)),
q_lora_rank=mla_dims.q_lora_rank,
kv_lora_rank=mla_dims.kv_lora_rank,
qk_nope_head_dim=mla_dims.qk_nope_head_dim,
qk_rope_head_dim=mla_dims.qk_rope_head_dim,
v_head_dim=mla_dims.v_head_dim,
)
)
init_workspace_manager(device)
batch_spec = BatchSpec(
seq_lens=[CONTEXT_LEN] * BATCH_SIZE,
query_lens=[DECODE_QLEN] * BATCH_SIZE,
)
captured: dict = {}
with set_current_vllm_config(vllm_config):
builder = builder_cls(spec, [layer_name], vllm_config, device)
common_attn_metadata = create_common_attn_metadata(
batch_spec, PAGE_SIZE, device, arange_block_indices=True
)
import aiter
real_get_mla_metadata_v1 = aiter.get_mla_metadata_v1
def spy(*args, **kwargs):
captured["args"] = args
captured["kwargs"] = dict(kwargs)
return real_get_mla_metadata_v1(*args, **kwargs)
with patch("aiter.get_mla_metadata_v1", spy):
metadata = builder.build(
common_prefix_len=0,
common_attn_metadata=common_attn_metadata,
)
return metadata, captured
def _compute_golden_metadata(captured: dict) -> dict[str, torch.Tensor]:
"""Recompute the persistent metadata with explicit fp8/bf16 dtypes.
Replays ``get_mla_metadata_v1`` on the builder's exact input tensors with
fresh output buffers and the explicitly-correct dtypes. This reference must
match the builder's output when the fix is in place.
"""
import aiter
args = captured["args"]
inputs = args[:_NUM_INPUT_ARGS]
# Fresh copies so the golden does not alias the builder's persistent buffers.
fresh_outputs = [arg.clone() for arg in args[_NUM_INPUT_ARGS:]]
golden_kwargs = dict(captured["kwargs"])
golden_kwargs["dtype_q"] = EXPECTED_Q_DTYPE
golden_kwargs["dtype_kv"] = EXPECTED_KV_DTYPE
aiter.get_mla_metadata_v1(*inputs, *fresh_outputs, **golden_kwargs)
return dict(zip(_OUTPUT_ARG_FIELDS, fresh_outputs))
def test_persistent_decode_metadata_matches_fp8_golden():
"""The builder's metadata must match the dtype-correct golden.
Regression guard: the fixed builder forwards fp8/bf16 dtypes so its
split/reduce metadata matches the golden recomputed with those explicit
dtypes. Dropping the dtypes (the original bug) produces a different layout
and fails this test.
"""
metadata, captured = _build_decode_metadata()
# qlen=1 must take the persistent-metadata path for this to be meaningful.
assert metadata.decode is not None
assert metadata.decode.has_persistent_metadata
assert metadata.work_meta_data is not None
golden = _compute_golden_metadata(captured)
mismatched = [
name
for name in _CONTENT_METADATA_FIELDS
if getattr(metadata, name).shape != golden[name].shape
or not torch.equal(getattr(metadata, name), golden[name])
]
assert not mismatched, (
"AITER MLA persistent decode metadata does not match the fp8/bf16 "
f"golden for fields {mismatched}; the builder must forward "
"dtype_q/dtype_kv to get_mla_metadata_v1."
)
@@ -1,246 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Accuracy test for the ROCm AITER FP8 MLA prefill path.
The FP8 MLA prefill path
(``AiterMLAImpl._mla_fp8_prefill_attn`` -> ``mla_prefill_ps_asm_fwd`` +
``mla_reduce_v1``) is auto-enabled on gfx950 (MI355) when AITER ships those
kernels, and it has no other test coverage. This test drives the *real*
metadata builder and impl methods (via ``object.__new__``, mirroring
``test_rocm_aiter_mla_head_padding.py::test_h12_aiter_mla_decode_matches_reference``)
so it exercises the actual persistent-scheduling metadata contract
(``get_ps_metadata_v1``) plus both kernels -- rather than re-implementing them --
and compares the output against a causal SDPA reference.
gfx950-only: ``mla_prefill_ps_asm_fwd`` has no gfx942 build, so this is skipped
on every other platform.
"""
import math
from types import SimpleNamespace
import pytest
import torch
import torch.nn.functional as F
from vllm.platforms import current_platform
def _fp8_prefill_available() -> bool:
if not (current_platform.is_rocm() and torch.cuda.is_available()):
return False
from vllm.v1.attention.backends.mla.rocm_aiter_mla import (
_fp8_mla_prefill_supported,
)
return _fp8_mla_prefill_supported()
pytestmark = pytest.mark.skipif(
not _fp8_prefill_available(),
reason="FP8 MLA prefill needs gfx950 (MI355) + an AITER build exporting "
"mla_prefill_ps_asm_fwd / mla_reduce_v1",
)
# DeepSeek-style MLA head dims after kv_b_proj decompression (q/k carry the
# nope+rope dims, v carries v_head_dim).
#
# The kernel requires 16-aligned heads, so _mla_fp8_prefill_attn replicate-pads
# q/k/v up to get_fp8_prefill_num_heads(num_heads) and slices the output back.
# Cover all four regimes:
# 8 -> padded to 16 (divisor, repeat_interleave), so the real heads sit at
# stride 2 in the padded output. Live case: Kimi-K2.x at TP8.
# 12 -> padded to 16 (non-divisor, tile+slice). Live case: a K3 rank at TP8.
# 16 -> no padding, output aliases the caller's buffer.
# 24 -> padded to 32. No current model and TP reaches this band (96 heads
# would need TP4, which does not fit in 288 GiB; 128-head models give
# 128/64/32/16/8), so this case exists to keep the general path honest
# for future architectures: it is the only one where padding above
# 16 actually happens, so it is what proves the replicate-pad/slice-back
# argument holds for a target other than 16.
HEAD_COUNTS = [8, 12, 16, 24]
QK_NOPE_HEAD_DIM = 128
QK_ROPE_HEAD_DIM = 64
QK_HEAD_DIM = QK_NOPE_HEAD_DIM + QK_ROPE_HEAD_DIM # 192
V_HEAD_DIM = 128
SCALE = 1.0 / math.sqrt(QK_HEAD_DIM)
# Dominated by fp8 (e4m3) rounding of q/k/v. Measured on MI355: worst-element
# abs error ~0.054, so atol=0.1 is ~2x that noise floor; rtol=0.05 bounds larger
# elements.
ATOL, RTOL = 1e-1, 5e-2
# The assembly kernel writes bf16 through a raw output pointer. Keep its tensors
# and workspace reservation on one dtype.
ATTN_OUT_DTYPE = torch.bfloat16
@pytest.fixture(autouse=True)
def _workspace_manager():
"""The FP8 prefill path draws per-call scratch from the global workspace
manager, which ``GPUModelRunner.__init__`` normally initializes. Set it up
for the test and tear it down so no global state leaks to other tests.
"""
from vllm.v1.worker.workspace import (
init_workspace_manager,
reset_workspace_manager,
)
init_workspace_manager(torch.device("cuda"))
yield
reset_workspace_manager()
def _make_impl(num_heads: int):
"""Minimal AiterMLAImpl exposing only what _mla_fp8_prefill_attn reads."""
from aiter import mla_prefill_ps_asm_fwd, mla_reduce_v1
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAImpl
impl = object.__new__(AiterMLAImpl)
impl.num_heads = num_heads
impl.v_head_dim = V_HEAD_DIM
impl.scale = SCALE
impl._mla_prefill_ps_asm_fwd = mla_prefill_ps_asm_fwd
impl._mla_reduce_v1 = mla_reduce_v1
return impl
def _build_prefill_metadata(
seq_lens: list[int],
device: torch.device,
num_heads: int,
attn_out_dtype: torch.dtype = ATTN_OUT_DTYPE,
):
"""Build metadata with the production builder and output dtype."""
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAMetadataBuilder
qo_indptr_cpu = torch.zeros(len(seq_lens) + 1, dtype=torch.int32)
qo_indptr_cpu[1:] = torch.tensor(seq_lens, dtype=torch.int32).cumsum(0)
qo_indptr = qo_indptr_cpu.to(device)
total_q = int(qo_indptr_cpu[-1].item())
max_q = max(seq_lens)
builder = object.__new__(AiterMLAMetadataBuilder)
builder.num_heads = num_heads
builder.mla_dims = SimpleNamespace(v_head_dim=V_HEAD_DIM)
builder._init_fp8_prefill_ps_buffers(
max_num_reqs=len(seq_lens),
max_prefill_qlen=max_q,
max_num_batched_tokens=total_q,
attn_out_dtype=attn_out_dtype,
device=device,
)
# SimpleNamespace stands in for the metadata dataclasses: the builder only
# reads prefill.{query_start_loc,max_query_len}, num_decodes, and
# common.query_start_loc_cpu, then populates metadata.fp8_prefill_*.
prefill = SimpleNamespace(query_start_loc=qo_indptr, max_query_len=max_q)
metadata = SimpleNamespace(prefill=prefill, num_decodes=0)
common = SimpleNamespace(query_start_loc_cpu=qo_indptr_cpu)
builder._build_fp8_prefill_ps_metadata(metadata, common)
return metadata, total_q, builder
def _reference(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor) -> torch.Tensor:
"""Causal SDPA reference on fp8-quantized inputs (matches kernel dtype)."""
fp8 = current_platform.fp8_dtype()
qf = q.to(fp8).float().transpose(0, 1) # [H, S, Dqk]
kf = k.to(fp8).float().transpose(0, 1)
vf = v.to(fp8).float().transpose(0, 1) # [H, S, Dv]
out = F.scaled_dot_product_attention(qf, kf, vf, is_causal=True, scale=SCALE)
return out.transpose(0, 1) # [S, H, Dv]
@pytest.mark.parametrize("num_heads", HEAD_COUNTS)
@pytest.mark.parametrize("seq_len", [128, 512])
@torch.inference_mode()
def test_fp8_prefill_matches_reference(seq_len: int, num_heads: int) -> None:
device = torch.device("cuda")
torch.manual_seed(0)
metadata, total_q, _ = _build_prefill_metadata([seq_len], device, num_heads)
# Lock the workspace, as GPUModelRunner does after warmup, so the forward
# cannot quietly grow it. The builder reserves scratch at num_head_k and
# the forward requests it at its own padded width; if a future change makes
# the forward's width exceed the builder's, that is an under-reservation
# which is invisible on an unlocked workspace (it just grows) and fatal in
# a real serve. Locking turns it into a failure here.
#
# Note this does not catch the reverse -- a builder wider than the forward
# merely over-reserves -- so it would not have flagged the pre-PR
# max(16, num_heads) sizing, which over-reserves 2.8x at 24 heads.
from vllm.v1.worker.workspace import lock_workspace
lock_workspace()
qkv_kwargs = dict(dtype=ATTN_OUT_DTYPE, device=device)
q = torch.randn(total_q, num_heads, QK_HEAD_DIM, **qkv_kwargs)
k = torch.randn(total_q, num_heads, QK_HEAD_DIM, **qkv_kwargs)
v = torch.randn(total_q, num_heads, V_HEAD_DIM, **qkv_kwargs)
out = torch.zeros(
total_q, num_heads * V_HEAD_DIM, dtype=ATTN_OUT_DTYPE, device=device
)
impl = _make_impl(num_heads)
impl._mla_fp8_prefill_attn(q, k, v, metadata, out)
out_ref = _reference(q, k, v)
assert torch.isfinite(out).all()
torch.testing.assert_close(
out.view(total_q, num_heads, V_HEAD_DIM).float(),
out_ref.float(),
atol=ATOL,
rtol=RTOL,
)
@pytest.mark.parametrize("num_heads", HEAD_COUNTS)
def test_fp8_prefill_metadata_width_matches_forward(num_heads: int) -> None:
"""The PS metadata must be built for the head count the kernel is handed.
``_init_fp8_prefill_ps_buffers``/``build()`` size the work and reduce maps
from ``num_head_k``, while ``_mla_fp8_prefill_attn`` pads q/k/v to its own
target. Divergence is not a numerical bug in either direction -- maps built
narrower than the kernel width still cover the real heads, since padding
only ever appends duplicate heads above them. It is a sizing bug: narrower
maps mean a lower head alignment, hence more partial tiles and a larger
reservation, while wider ones under-reserve the forward's scratch. Pin the
two to one helper rather than to independent expressions.
"""
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAHelper
width = AiterMLAHelper.get_fp8_prefill_num_heads(num_heads)
assert width % AiterMLAHelper._AITER_MIN_MLA_HEADS == 0
assert width >= num_heads
assert width - num_heads < AiterMLAHelper._AITER_MIN_MLA_HEADS
# Every count the enable gate accepts must have a usable padded width.
assert AiterMLAHelper.is_valid_num_heads(num_heads)
# The builder must actually size its maps at that width. Assert against the
# buffer it allocated rather than recomputing the width here, or the test
# passes no matter what _init_fp8_prefill_ps_buffers does.
#
# fp8_ps_reduce_partial_map is sized by the partial-tile count, which is
# gcd-driven and so is a sharp probe of the width used: a lower head
# alignment yields *more* tiles. At 24 heads, sizing at the unpadded count
# gives 4x the tiles (64 vs 16) and ~2.8x the scratch reservation (193.6
# vs 68.6 MiB at batch=1/qlen=512) -- not wrong numerically (the 24-wide
# maps still cover the real heads) but the reason to pin the widths
# together.
from aiter import get_ps_metadata_info_v1
from vllm.v1.attention.backends.mla.rocm_aiter_mla import _FP8_PREFILL_TILE_Q
def _tiles(nhk: int) -> int:
return get_ps_metadata_info_v1(
batch_size=1,
num_head_k=nhk,
max_qlen=512,
qlen_granularity=_FP8_PREFILL_TILE_Q,
)[5][0]
_, _, builder = _build_prefill_metadata([512], torch.device("cuda"), num_heads)
assert builder.fp8_ps_reduce_partial_map.numel() == _tiles(width)
@@ -1,252 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for ROCm AITER MLA FP8 support detection."""
import sys
import types
from typing import Any
from unittest.mock import patch
import pytest
from vllm.platforms import current_platform
_SKIP_UNSUPPORTED_AITER_HARDWARE = True
if current_platform.is_rocm():
from vllm.platforms.rocm import get_cdna_version
_SKIP_UNSUPPORTED_AITER_HARDWARE = get_cdna_version() <= 2
pytestmark = pytest.mark.skipif(
not current_platform.is_rocm(), reason="ROCm-specific tests"
)
@pytest.fixture(autouse=True)
def reset_aiter_mla_support_cache(monkeypatch: pytest.MonkeyPatch) -> None:
import vllm._aiter_ops as aiter_ops
monkeypatch.setattr(aiter_ops, "_AITER_MLA_SUPPORTS_FP8", None)
def _install_fake_aiter_modules(
monkeypatch: pytest.MonkeyPatch,
*,
supports_fp8: bool,
supports_causal: bool = False,
) -> None:
aiter_mod: Any = types.ModuleType("aiter")
mla_mod: Any = types.ModuleType("aiter.mla")
if supports_fp8:
def mla_decode_fwd_with_fp8(
q,
kv_buffer,
kv_indptr,
kv_indices,
o,
sm_scale,
q_scale=None,
kv_scale=None,
):
return None
mla_decode_fwd: Any = mla_decode_fwd_with_fp8
else:
def mla_decode_fwd_without_fp8(
q,
kv_buffer,
kv_indptr,
kv_indices,
o,
sm_scale,
):
return None
mla_decode_fwd = mla_decode_fwd_without_fp8
if supports_causal:
inner = mla_decode_fwd
def mla_decode_fwd_with_causal(*args, causal=True, **kwargs):
return inner(*args, **kwargs)
mla_decode_fwd = mla_decode_fwd_with_causal
mla_mod.mla_decode_fwd = mla_decode_fwd
aiter_mod.mla = mla_mod
monkeypatch.setitem(sys.modules, "aiter", aiter_mod)
monkeypatch.setitem(sys.modules, "aiter.mla", mla_mod)
def test_aiter_mla_fp8_support_detects_fp8_signature(monkeypatch):
"""The support check should detect q_scale and kv_scale parameters."""
from vllm._aiter_ops import _check_aiter_mla_fp8_support
_install_fake_aiter_modules(monkeypatch, supports_fp8=True)
assert _check_aiter_mla_fp8_support() is True
def test_aiter_mla_fp8_support_rejects_missing_fp8_signature(monkeypatch):
"""The support check should return False when FP8 params are absent."""
from vllm._aiter_ops import _check_aiter_mla_fp8_support
_install_fake_aiter_modules(monkeypatch, supports_fp8=False)
assert _check_aiter_mla_fp8_support() is False
@pytest.mark.skipif(
_SKIP_UNSUPPORTED_AITER_HARDWARE,
reason="Installed AITER MLA FP8 check requires CDNA 3 or newer",
)
def test_installed_aiter_mla_supports_fp8():
"""Supported ROCm CI must provide AITER with MLA FP8 scaling."""
from vllm._aiter_ops import (
_check_aiter_mla_fp8_support,
is_aiter_found_and_supported,
)
assert is_aiter_found_and_supported(), (
"AITER must be installed on supported ROCm hardware"
)
assert _check_aiter_mla_fp8_support() is True
@pytest.mark.parametrize(
"error_type",
[ImportError, ModuleNotFoundError, AttributeError, ValueError, TypeError],
)
def test_aiter_mla_fp8_support_handles_signature_errors(monkeypatch, error_type):
"""The support check should fail closed on import or signature problems."""
import vllm._aiter_ops as aiter_ops
from vllm._aiter_ops import _check_aiter_mla_fp8_support
_install_fake_aiter_modules(monkeypatch, supports_fp8=True)
with patch("inspect.signature", side_effect=error_type("boom")):
assert _check_aiter_mla_fp8_support() is False
assert aiter_ops._AITER_MLA_SUPPORTS_FP8 is False
def test_aiter_mla_fp8_support_result_is_cached(monkeypatch):
"""The support check should reuse the cached result on later calls."""
import inspect
from vllm._aiter_ops import _check_aiter_mla_fp8_support
_install_fake_aiter_modules(monkeypatch, supports_fp8=True)
with patch("inspect.signature", wraps=inspect.signature) as signature_mock:
assert _check_aiter_mla_fp8_support() is True
assert _check_aiter_mla_fp8_support() is True
assert signature_mock.call_count == 1
def test_missing_causal_arg_fails_closed(monkeypatch):
"""Old aiter wheels have no causal=; do not report that as a valid draft path."""
from vllm._aiter_ops import rocm_aiter_ops
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLABackend
_install_fake_aiter_modules(monkeypatch, supports_fp8=True, supports_causal=False)
with pytest.raises(RuntimeError, match="causal-only"):
rocm_aiter_ops.mla_decode_supports_non_causal()
with pytest.raises(RuntimeError, match="causal-only"):
AiterMLABackend.supports_non_causal()
@pytest.mark.parametrize("has_kernels", [False, True])
def test_supports_non_causal_requires_arch_kernels(monkeypatch, has_kernels):
"""The Python argument alone does not guarantee an arch has the kernel."""
from vllm._aiter_ops import rocm_aiter_ops
from vllm.v1.attention.backends.mla import rocm_aiter_mla
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLABackend
_install_fake_aiter_modules(monkeypatch, supports_fp8=True, supports_causal=True)
monkeypatch.setattr(
rocm_aiter_mla,
"_aiter_mla_non_causal_asm_kernels",
lambda: has_kernels,
)
assert rocm_aiter_ops.mla_decode_supports_non_causal() is True
assert AiterMLABackend.supports_non_causal() is has_kernels
@pytest.mark.parametrize("causal", [True, False])
def test_decode_fwd_forwards_causal_to_aiter(monkeypatch, causal):
"""The decode op must pass the block mask through; omitting it is causal-only."""
seen: list = []
def mla_decode_fwd(*args, causal=None, **kwargs):
seen.append(causal)
mla_mod: Any = types.ModuleType("aiter.mla")
mla_mod.mla_decode_fwd = mla_decode_fwd
aiter_mod: Any = types.ModuleType("aiter")
aiter_mod.mla = mla_mod
monkeypatch.setitem(sys.modules, "aiter", aiter_mod)
monkeypatch.setitem(sys.modules, "aiter.mla", mla_mod)
import torch
import vllm._aiter_ops as aiter_ops
aiter_ops._rocm_aiter_mla_decode_fwd_impl(
torch.zeros(2, 1, 8),
torch.zeros(2, 1, 1, 8),
torch.zeros(2, 1, 8),
torch.zeros(3, dtype=torch.int32),
1,
causal=causal,
)
assert seen == [causal]
def _aiter_mla_validate(
monkeypatch,
*,
dtype,
kv_cache_dtype,
use_non_causal,
):
from vllm.platforms.interface import DeviceCapability
from vllm.v1.attention.backend import AttentionType
from vllm.v1.attention.backends.mla import rocm_aiter_mla
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLABackend
_install_fake_aiter_modules(monkeypatch, supports_fp8=True, supports_causal=True)
monkeypatch.setattr(
rocm_aiter_mla, "_aiter_mla_non_causal_asm_kernels", lambda: True
)
return AiterMLABackend.validate_configuration(
head_size=576,
dtype=dtype,
kv_cache_dtype=kv_cache_dtype,
block_size=1,
use_mla=True,
has_sink=False,
use_sparse=False,
use_mm_prefix=False,
use_per_head_quant_scales=False,
device_capability=DeviceCapability(9, 5),
attn_type=AttentionType.DECODER,
use_non_causal=use_non_causal,
)
def test_non_causal_fp16_auto_cache_is_rejected(monkeypatch):
"""Pinned AITER aborts fp16 Q; auto-select must fall through to Triton."""
import torch
reasons = _aiter_mla_validate(
monkeypatch,
dtype=torch.float16,
kv_cache_dtype="auto",
use_non_causal=True,
)
assert any("non-causal fp16" in reason for reason in reasons)
@@ -1,357 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Head padding and kernel selection for the ROCm AITER MLA backend.
The asm persistent decode requires a 16-aligned head count, so unaligned
counts through 128 are tile-padded to the next multiple of 16 and sliced back
off the output. Small divisor counts (1/2/4/8) preserve their existing
repeat-interleave path and may keep the Gluon kernel on gfx950.
"""
import math
import types
import pytest
import torch
import torch.nn.functional as F
from vllm._aiter_ops import is_aiter_found
from vllm.platforms import current_platform
from vllm.v1.attention.backends.mla import rocm_aiter_mla
from vllm.v1.attention.backends.mla.rocm_aiter_mla import (
AiterMLADecodeMetadata,
AiterMLAHelper,
AiterMLAImpl,
AiterMLAMetadata,
)
NUM_HEADS = 12
KV_LORA_RANK = 512
QK_ROPE_HEAD_DIM = 64
QK_HEAD_DIM = KV_LORA_RANK + QK_ROPE_HEAD_DIM
CONTEXT_LEN = 4096
SCALE = 1.0 / math.sqrt(QK_HEAD_DIM)
# Small non-divisor counts use tile-and-slice; divisors of 16 keep
# repeat_interleave. Both pad to exactly 16 and round-trip.
NON_DIVISOR_HEADS = [3, 5, 6, 7, 9, 10, 11, 12, 13, 14, 15]
DIVISOR_HEADS = [1, 2, 4, 8]
@pytest.fixture(autouse=True)
def _disable_native_h24(monkeypatch):
"""Exercise the padding fallback unless a test explicitly enables H24."""
monkeypatch.setattr(
rocm_aiter_mla, "_aiter_mla_native_h24_supported", lambda: False
)
def _rocm_aiter_available() -> bool:
return current_platform.is_rocm() and is_aiter_found() and torch.cuda.is_available()
def _on_gfx950() -> bool:
if not (current_platform.is_rocm() and is_aiter_found()):
return False
from vllm.platforms.rocm import on_gfx950
return on_gfx950()
def _expected_tile_pad(q: torch.Tensor, num_heads: int, m: int = 16) -> torch.Tensor:
# get_mla_padded_q tiles the heads and slices to m, i.e. head i of the
# padded tensor is head (i % num_heads) of the input.
idx = [i % num_heads for i in range(m)]
return q[:, idx, :]
def _make_h12_decode_metadata(device: torch.device) -> AiterMLAMetadata:
page_indices = torch.arange(CONTEXT_LEN, dtype=torch.int32, device=device)
decode = AiterMLADecodeMetadata(
block_table=page_indices.view(1, -1),
seq_lens=torch.tensor([CONTEXT_LEN], dtype=torch.int32, device=device),
dcp_tot_seq_lens=None,
paged_kv_indptr=torch.tensor(
[0, CONTEXT_LEN], dtype=torch.int32, device=device
),
paged_kv_indices=page_indices,
paged_kv_last_page_len=torch.ones(1, dtype=torch.int32, device=device),
qo_indptr=torch.tensor([0, 1], dtype=torch.int32, device=device),
attn_out_dtype=torch.bfloat16,
max_qo_len=1,
)
return AiterMLAMetadata(
num_reqs=1,
max_query_len=1,
max_seq_len=CONTEXT_LEN,
num_actual_tokens=1,
query_start_loc=decode.qo_indptr,
slot_mapping=torch.tensor([CONTEXT_LEN - 1], dtype=torch.int64, device=device),
num_decodes=1,
num_decode_tokens=1,
num_prefills=0,
decode=decode,
)
def test_h12_query_is_tile_padded_to_h16():
q = torch.arange(2 * 12 * 4, dtype=torch.bfloat16).view(2, 12, 4)
padded_q = AiterMLAHelper.get_mla_padded_q(12, q)
assert padded_q.shape == (2, 16, 4)
assert padded_q.is_contiguous()
# The real heads are untouched...
torch.testing.assert_close(padded_q[:, :12], q)
# ...and the 4 padding heads are the tiled wrap-around (heads 0..3), not
# zeros: MLA attention is per-head independent so duplicate query heads are
# harmless and get sliced back off the output.
torch.testing.assert_close(padded_q[:, 12:], q[:, :4])
def test_h6_tp16_query_is_padded_to_h16():
# TP16 puts 6 heads/rank. The old append-only padding produced cat(6, 6) =
# 12 heads and broke the asm kernel; tile-and-slice must reach exactly 16.
q = torch.arange(2 * 6 * 4, dtype=torch.bfloat16).view(2, 6, 4)
padded_q = AiterMLAHelper.get_mla_padded_q(6, q)
assert padded_q.shape == (2, 16, 4)
assert padded_q.is_contiguous()
torch.testing.assert_close(padded_q, _expected_tile_pad(q, 6))
def test_h12_output_discards_padding_heads():
o = torch.arange(2 * 16 * 4, dtype=torch.bfloat16).view(2, 16, 4)
unpadded_o = AiterMLAHelper.get_mla_unpadded_o(12, o)
assert unpadded_o.shape == (2, 12, 4)
torch.testing.assert_close(unpadded_o, o[:, :12])
def test_h24_query_is_tile_padded_to_h32():
q = torch.arange(2 * 24 * 4, dtype=torch.float32).view(2, 24, 4)
padded_q = AiterMLAHelper.get_mla_padded_q(24, q)
assert padded_q.shape == (2, 32, 4)
assert padded_q.is_contiguous()
torch.testing.assert_close(padded_q[:, :24], q)
torch.testing.assert_close(padded_q[:, 24:], q[:, :8])
def test_h24_output_discards_h32_padding_heads():
o = torch.arange(2 * 32 * 4, dtype=torch.float32).view(2, 32, 4)
unpadded_o = AiterMLAHelper.get_mla_unpadded_o(24, o)
assert unpadded_o.shape == (2, 24, 4)
torch.testing.assert_close(unpadded_o, o[:, :24])
def test_h24_reducer_without_metadata_still_pads_to_h32(monkeypatch):
monkeypatch.setattr(
rocm_aiter_mla, "_aiter_mla_native_h24_reducer_supported", lambda: True
)
monkeypatch.setattr(
rocm_aiter_mla, "_aiter_mla_native_h24_metadata_supported", lambda: False
)
monkeypatch.setattr(
rocm_aiter_mla,
"_aiter_mla_native_h24_supported",
lambda: (
rocm_aiter_mla._aiter_mla_native_h24_reducer_supported()
and rocm_aiter_mla._aiter_mla_native_h24_metadata_supported()
),
)
q = torch.arange(2 * 24 * 4, dtype=torch.float32).view(2, 24, 4)
assert AiterMLAHelper.get_actual_mla_num_heads(24) == 32
padded_q = AiterMLAHelper.get_mla_padded_q(24, q)
assert padded_q.shape == (2, 32, 4)
torch.testing.assert_close(padded_q[:, :24], q)
torch.testing.assert_close(padded_q[:, 24:], q[:, :8])
def test_native_h24_requires_reducer_and_metadata(monkeypatch):
monkeypatch.setattr(
rocm_aiter_mla, "_aiter_mla_native_h24_reducer_supported", lambda: True
)
monkeypatch.setattr(
rocm_aiter_mla, "_aiter_mla_native_h24_metadata_supported", lambda: True
)
monkeypatch.setattr(
rocm_aiter_mla,
"_aiter_mla_native_h24_supported",
lambda: (
rocm_aiter_mla._aiter_mla_native_h24_reducer_supported()
and rocm_aiter_mla._aiter_mla_native_h24_metadata_supported()
),
)
q = torch.arange(2 * 24 * 4, dtype=torch.float32).view(2, 24, 4)
assert AiterMLAHelper.get_actual_mla_num_heads(24) == 24
assert AiterMLAHelper.get_mla_padded_q(24, q) is q
assert AiterMLAHelper.get_mla_unpadded_o(24, q) is q
def test_existing_divisor_head_mapping_is_unchanged():
q = torch.arange(2 * 8 * 4, dtype=torch.bfloat16).view(2, 8, 4)
padded_q = AiterMLAHelper.get_mla_padded_q(8, q)
unpadded_o = AiterMLAHelper.get_mla_unpadded_o(8, padded_q)
# Divisor counts still use repeat_interleave / strided unpad, unchanged.
torch.testing.assert_close(padded_q, q.repeat_interleave(2, dim=1))
torch.testing.assert_close(unpadded_o, q)
@pytest.mark.parametrize("num_heads", [17, 24, 31])
def test_unaligned_head_counts_round_trip_through_h32(num_heads: int):
q = torch.arange(2 * num_heads * 4, dtype=torch.float32).view(2, num_heads, 4)
padded_q = AiterMLAHelper.get_mla_padded_q(num_heads, q)
unpadded_o = AiterMLAHelper.get_mla_unpadded_o(num_heads, padded_q)
assert padded_q.shape == (2, 32, 4)
assert padded_q.is_contiguous()
torch.testing.assert_close(unpadded_o, q)
@pytest.mark.parametrize("num_heads", NON_DIVISOR_HEADS + DIVISOR_HEADS)
def test_all_small_head_counts_pad_to_16_and_round_trip(num_heads: int):
q = torch.arange(2 * num_heads * 4, dtype=torch.float32).view(2, num_heads, 4)
padded_q = AiterMLAHelper.get_mla_padded_q(num_heads, q)
unpadded_o = AiterMLAHelper.get_mla_unpadded_o(num_heads, padded_q)
assert padded_q.shape == (2, 16, 4)
assert padded_q.is_contiguous()
# The real heads survive the pad -> unpad round trip exactly.
torch.testing.assert_close(unpadded_o, q)
def test_aligned_h32_is_zero_copy():
q = torch.arange(2 * 32 * 4, dtype=torch.float32).view(2, 32, 4)
assert AiterMLAHelper.get_mla_padded_q(32, q) is q
assert AiterMLAHelper.get_mla_unpadded_o(32, q) is q
def test_is_valid_num_heads():
for n in range(1, 129):
assert AiterMLAHelper.is_valid_num_heads(n)
assert AiterMLAHelper.is_valid_num_heads(24)
assert AiterMLAHelper.is_valid_num_heads(127)
# Aligned counts remain valid above the range where padding is supported.
assert AiterMLAHelper.is_valid_num_heads(144)
assert not AiterMLAHelper.is_valid_num_heads(0)
assert not AiterMLAHelper.is_valid_num_heads(129)
def test_nondivisor_and_multitoken_never_use_gluon():
# Non-divisor decode always takes the asm path (12 heads/rank at TP8).
assert not AiterMLAHelper.use_gluon_decode(12, 1, "auto")
assert not AiterMLAHelper.use_gluon_decode(6, 1, "auto")
# >=16 heads never use Gluon, including unaligned counts padded for asm.
assert not AiterMLAHelper.use_gluon_decode(16, 1, "auto")
assert not AiterMLAHelper.use_gluon_decode(24, 1, "auto")
# Multi-token (verify / qlen>1) is never the single-token Gluon decode.
assert not AiterMLAHelper.use_gluon_decode(8, 4, "auto")
assert not AiterMLAHelper.use_gluon_decode(12, 4, "auto")
def test_divisor_gluon_selection_follows_arch():
# Divisor head counts keep Gluon only where the kernel exists (gfx950).
# On gfx942 / non-ROCm they must route to the asm persistent decode.
on_gfx950 = _on_gfx950()
assert AiterMLAHelper.use_gluon_decode(8, 1, "auto") is on_gfx950
assert AiterMLAHelper.use_gluon_decode(4, 1, "auto") is on_gfx950
assert AiterMLAHelper.use_gluon_decode(1, 1, "auto") is on_gfx950
def test_asm_padding_env_default_is_auto(monkeypatch):
monkeypatch.delenv("VLLM_ROCM_AITER_MLA_ASM_PADDING", raising=False)
import vllm.envs as envs
assert envs.VLLM_ROCM_AITER_MLA_ASM_PADDING == "auto"
def test_asm_padding_env_force_asm_disables_gluon(monkeypatch):
monkeypatch.setenv("VLLM_ROCM_AITER_MLA_ASM_PADDING", "asm")
# Forcing the asm path: no small-head count uses Gluon on any arch.
for num_heads in (1, 2, 4, 8, 12):
assert not AiterMLAHelper.use_gluon_decode(num_heads, 1, "auto")
def test_asm_padding_env_force_gluon_follows_arch(monkeypatch):
monkeypatch.setenv("VLLM_ROCM_AITER_MLA_ASM_PADDING", "gluon")
on_gfx950 = _on_gfx950()
# Forcing Gluon: any 1..15 single-token decode uses it where a build exists
# (gfx950), including non-divisor counts like 12; gfx942/non-ROCm still
# falls back to the asm path.
for num_heads in (1, 2, 4, 8, 12):
assert AiterMLAHelper.use_gluon_decode(num_heads, 1, "auto") is on_gfx950
def test_asm_padding_env_auto_matches_arch_gate(monkeypatch):
monkeypatch.setenv("VLLM_ROCM_AITER_MLA_ASM_PADDING", "auto")
on_gfx950 = _on_gfx950()
# auto: divisor counts keep Gluon on gfx950, non-divisor counts take asm.
assert AiterMLAHelper.use_gluon_decode(8, 1, "auto") is on_gfx950
assert not AiterMLAHelper.use_gluon_decode(12, 1, "auto")
@pytest.mark.skipif(
not _rocm_aiter_available(),
reason="12-head AITER MLA asm persistent decode needs ROCm + AITER",
)
@torch.inference_mode()
def test_h12_aiter_mla_decode_matches_reference():
"""The 12-head AITER decode output must match an attention reference.
12 is a non-divisor of 16 so use_gluon_decode is False on every arch; the
query is tile-padded to 16 heads and served by the asm persistent decode
(the CDNA3 path this PR enables, and unchanged on CDNA4).
"""
device = torch.device("cuda:0")
torch.manual_seed(42)
q = torch.randn(1, NUM_HEADS, QK_HEAD_DIM, dtype=torch.bfloat16, device=device)
kv_cache = torch.randn(
CONTEXT_LEN, 1, QK_HEAD_DIM, dtype=torch.bfloat16, device=device
)
impl = object.__new__(AiterMLAImpl)
impl.num_heads = NUM_HEADS
impl.dcp_world_size = 1
impl.kv_lora_rank = KV_LORA_RANK
impl.qk_rope_head_dim = QK_ROPE_HEAD_DIM
impl.scale = SCALE
impl.kv_cache_dtype = "auto"
one = torch.ones(1, dtype=torch.float32, device=device)
layer = types.SimpleNamespace(_q_scale=one, _k_scale=one)
metadata = _make_h12_decode_metadata(device)
out, _ = impl.forward_mqa(q, kv_cache, metadata, layer)
assert out.shape[1] == NUM_HEADS # padding heads sliced back off
key = kv_cache[:, 0].float().unsqueeze(0)
value = key[..., :KV_LORA_RANK]
out_ref = F.scaled_dot_product_attention(
q[0].float().unsqueeze(1),
key,
value,
scale=SCALE,
enable_gqa=True,
).squeeze(1)
assert torch.isfinite(out).all()
torch.testing.assert_close(
out.float(),
out_ref.unsqueeze(0),
atol=1e-2,
rtol=1e-2,
)
@@ -1,523 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Correctness tests for ROCm AITER sparse MLA attention sinks."""
from types import SimpleNamespace
import pytest
import torch
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
pytestmark = pytest.mark.skipif(
not current_platform.is_rocm(), reason="ROCm-specific tests"
)
Q_HEAD_DIM = 576
V_HEAD_DIM = 512
SM_SCALE = Q_HEAD_DIM**-0.5
def _gamma_fp32(operations: int) -> float:
u = torch.finfo(torch.float32).eps / 2
return operations * u / (1 - operations * u)
def _sink_reference(q, keys, sinks, scale):
# Reference the stored operands, so input quantization is not kernel error.
q, keys, sinks = q.double(), keys.double(), sinks.double()
scores = q @ keys.T * scale
logits = torch.cat((scores, sinks[:, None]), dim=-1)
probabilities = logits.softmax(dim=-1)[:, :-1]
values = keys[:, :V_HEAD_DIM]
expected = probabilities @ values
value_scale = probabilities @ values.abs()
# FP32 dot accumulation and conversion/multiplication of the scale.
score_error = _gamma_fp32(q.shape[-1] + 2) * (q.abs() @ keys.abs().T) * abs(scale)
score_error = score_error.amax(dim=-1) if keys.shape[0] else torch.zeros_like(sinks)
return expected, logits.logsumexp(dim=-1), value_scale, score_error
def _assert_sink_output_close(
actual, expected, value_scale, score_error, probability_dtype, native
):
assert actual.shape == expected.shape
# Budget the identifiable rounding stages against the absolute value
# contributions: output-relative ULPs become unbounded when values cancel.
# Rounding P before P@V costs u(P) * sum(P*abs(V)).
# AITER rounds output before and after sink scaling; Triton rounds once.
u_probability = torch.finfo(probability_dtype).eps / 2
u_output = torch.finfo(actual.dtype).eps / 2
output_rounds = 2 if native else 1
amplification = (1 + u_output) ** output_rounds
relative_error = (1 + u_probability) * torch.exp(2 * score_error) - 1
allowance = (
relative_error.unsqueeze(-1) * value_scale * amplification
+ (amplification - 1) * expected.abs()
+ output_rounds * torch.finfo(actual.dtype).tiny * u_output
)
actual = actual.to(device=expected.device, dtype=torch.float64)
assert torch.isfinite(actual).all()
assert torch.count_nonzero(actual[value_scale == 0]) == 0
error = (actual - expected).abs()
max_ratio = (error / allowance.clamp_min(1e-300)).max().item()
assert max_ratio <= 1, f"exceeded rounding budget by {max_ratio:.3f}x"
def _require_aiter() -> None:
from vllm._aiter_ops import is_aiter_found_and_supported
from vllm.platforms.rocm import get_cdna_version
if get_cdna_version() not in (3, 4):
pytest.skip("AITER MLA requires CDNA 3 or 4")
if not is_aiter_found_and_supported():
pytest.skip("aiter is required on supported ROCm hardware for this test")
@pytest.mark.parametrize(
("real_heads", "cache_kind"),
[
(1, "bf16"),
(4, "bf16"),
(8, "bf16"),
(8, "fp8"),
(32, "fp8"),
(64, "fp8"),
(12, "bf16"),
(16, "bf16"),
(24, "bf16"),
(32, "bf16"),
(40, "bf16"),
(48, "bf16"),
(64, "bf16"),
(80, "bf16"),
],
)
@torch.inference_mode()
def test_sparse_mla_sink_matches_ragged_reference(
real_heads: int, cache_kind: str
) -> None:
"""Exercise ragged sink decode, head padding, and FP8 scale forwarding."""
_require_aiter()
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAHelper
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import (
ROCMAiterMLASparseImpl,
)
set_random_seed(real_heads * 17 + (cache_kind == "fp8"))
device = torch.device("cuda")
seq_lens = [0, 1, 5, 23, 17]
batch_size = len(seq_lens)
q_source = torch.randn(batch_size, real_heads, Q_HEAD_DIM, device=device) * 0.2
pool_size = sum(seq_lens) + 52
kv_source = torch.randn(pool_size, 1, Q_HEAD_DIM, device=device) * 0.2
if cache_kind == "fp8":
fp8_dtype = current_platform.fp8_dtype()
q_scale = torch.tensor(0.5, dtype=torch.float32, device=device)
kv_scale = torch.tensor(0.25, dtype=torch.float32, device=device)
q_real = (q_source / q_scale).to(fp8_dtype)
kv = (kv_source / kv_scale).to(fp8_dtype)
q_ref = q_real.float() * q_scale
kv_ref = kv.float() * kv_scale
else:
q_scale = kv_scale = None
q_real = q_source.to(torch.bfloat16)
kv = kv_source.to(torch.bfloat16)
q_ref = q_real.float()
kv_ref = kv.float()
q = AiterMLAHelper.get_mla_padded_q(real_heads, q_real)
indices = torch.randperm(pool_size, device=device)[: sum(seq_lens)].to(torch.int32)
kv_indptr = torch.tensor(
[0] + [sum(seq_lens[:i]) for i in range(1, len(seq_lens) + 1)],
dtype=torch.int32,
device=device,
)
# Non-None garbage proves the sink path cannot accidentally select the
# gfx942 persistent kernel, which has no return-LSE code object.
metadata = SimpleNamespace(
attn_out_dtype=torch.bfloat16,
qo_indptr=torch.arange(batch_size + 1, dtype=torch.int32, device=device),
paged_kv_indptr=kv_indptr,
paged_kv_indices=indices,
paged_kv_last_page_len=torch.ones(batch_size, dtype=torch.int32, device=device),
work_meta_data=torch.tensor([123], dtype=torch.int32),
work_indptr=None,
work_info_set=None,
reduce_indptr=None,
reduce_final_map=None,
reduce_partial_map=None,
num_prefills=0,
num_decodes=batch_size,
num_decode_tokens=batch_size,
max_query_len=1,
max_seq_len=max(seq_lens),
topk_tokens=max(seq_lens),
)
sinks = torch.linspace(-2.0, 6.0, real_heads, device=device)
impl = object.__new__(ROCMAiterMLASparseImpl)
impl.num_heads = real_heads
impl.kv_lora_rank = V_HEAD_DIM
impl.kv_cache_dtype = cache_kind
impl.scale = SM_SCALE
impl.sinks = sinks
output, lse = impl._forward_mla(
SimpleNamespace(_q_scale=q_scale, _k_scale=kv_scale),
q,
kv,
metadata,
)
kv_flat = kv_ref[:, 0]
references = []
start = 0
for batch_idx, seq_len in enumerate(seq_lens):
rows = kv_flat[indices[start : start + seq_len].long()]
references.append(_sink_reference(q_ref[batch_idx], rows, sinks, SM_SCALE))
start += seq_len
expected, expected_lse, value_scale, score_error = (
torch.stack(values) for values in zip(*references)
)
assert output.dtype == torch.bfloat16
if lse is not None:
assert lse.dtype == torch.float32
assert lse.shape == expected_lse.shape
lse_error = (lse.double() - expected_lse).abs()
rounded_lse = expected_lse.float()
positive_inf = torch.full_like(rounded_lse, float("inf"))
ulp = torch.maximum(
torch.nextafter(rounded_lse, positive_inf) - rounded_lse,
rounded_lse - torch.nextafter(rounded_lse, -positive_inf),
).double()
lse_allowance = score_error + 2 * ulp
assert torch.all(lse_error <= lse_allowance)
torch.testing.assert_close(lse[0].double(), expected_lse[0], atol=0, rtol=0)
else:
from vllm.platforms.rocm import on_gfx942
assert cache_kind == "bf16" and real_heads in (40, 48, 64) and on_gfx942()
_assert_sink_output_close(
output,
expected,
value_scale,
score_error,
q_real.dtype,
native=lse is not None,
)
@pytest.mark.parametrize(
("q_dtype", "kv_dtype"),
[
(torch.float16, torch.bfloat16),
(torch.bfloat16, torch.float16),
],
)
def test_sparse_mla_sink_rejects_unsupported_aiter_dtypes(
q_dtype: torch.dtype, kv_dtype: torch.dtype
) -> None:
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import (
ROCMAiterMLASparseImpl,
)
impl = object.__new__(ROCMAiterMLASparseImpl)
impl.num_heads = 16
impl.kv_lora_rank = V_HEAD_DIM
impl.kv_cache_dtype = "auto"
impl.scale = SM_SCALE
impl.sinks = torch.zeros(16, dtype=torch.float32)
metadata = SimpleNamespace(
attn_out_dtype=torch.bfloat16,
num_prefills=0,
num_decodes=1,
num_decode_tokens=1,
max_query_len=1,
)
with pytest.raises(ValueError, match="both use BF16 or both use FP8"):
impl._forward_mla(
SimpleNamespace(_q_scale=None, _k_scale=None),
torch.empty(1, 16, Q_HEAD_DIM, dtype=q_dtype),
torch.empty(1, 1, Q_HEAD_DIM, dtype=kv_dtype),
metadata,
)
def _make_noncontiguous_sink() -> torch.Tensor:
return torch.empty(8, dtype=torch.float32)[::2]
@pytest.mark.parametrize(
("sinks", "match"),
[
(torch.empty(4, dtype=torch.bfloat16), "must be float32"),
(torch.empty(2, 2, dtype=torch.float32), "must have shape"),
(_make_noncontiguous_sink(), "must be contiguous"),
],
)
def test_sparse_mla_sink_validation(sinks: torch.Tensor, match: str) -> None:
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import (
ROCMAiterMLASparseImpl,
)
with pytest.raises(ValueError, match=match):
ROCMAiterMLASparseImpl(
num_heads=4,
head_size=Q_HEAD_DIM,
scale=SM_SCALE,
num_kv_heads=1,
alibi_slopes=None,
sliding_window=None,
kv_cache_dtype="bfloat16",
logits_soft_cap=None,
attn_type="decoder",
kv_sharing_target_layer_name=None,
sinks=sinks,
kv_lora_rank=V_HEAD_DIM,
)
def test_sparse_mla_backend_reports_sink_support_for_current_hardware() -> None:
from vllm.platforms.rocm import get_cdna_version
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import (
ROCMAiterMLASparseBackend,
)
assert ROCMAiterMLASparseBackend.supports_sink() == (get_cdna_version() in (3, 4))
def test_sparse_mla_backend_rejects_dcp() -> None:
from vllm.platforms.rocm import RocmPlatform
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.attention.selector import AttentionSelectorConfig
selector_config = AttentionSelectorConfig(
head_size=Q_HEAD_DIM,
dtype=torch.bfloat16,
kv_cache_dtype="bfloat16",
block_size=16,
use_mla=True,
has_sink=True,
use_sparse=True,
use_mm_prefix=False,
use_per_head_quant_scales=False,
attn_type="decoder",
use_dcp=True,
)
with pytest.raises(ValueError, match="DCP not supported"):
RocmPlatform.get_attn_backend_cls(
selected_backend=AttentionBackendEnum.ROCM_AITER_MLA_SPARSE,
attn_selector_config=selector_config,
)
@pytest.mark.parametrize("num_heads,block_size", [(8, 16), (20, 32)])
@pytest.mark.parametrize("interleaved_pages", [False, True])
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@torch.inference_mode()
def test_sparse_mla_sink_matches_dense_attention_with_empty_rows_and_paged_cache(
num_heads: int,
block_size: int,
interleaved_pages: bool,
dtype: torch.dtype,
) -> None:
"""Preserve head-specific sinks and latent values across ragged cache pages."""
_require_aiter()
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAHelper
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import (
ROCMAiterMLASparseImpl,
)
from vllm.v1.attention.backends.mla.sparse_utils import flat_kv_row_view
head_dim, value_dim = 576, 512
generator = torch.Generator().manual_seed(421)
# The empty row exercises sink-only attention; the last row crosses both
# page boundaries and the kernel's 16-key reduction tile.
selected_rows = [
[],
[(2, 7)],
[(3, 1), (0, block_size - 1), (1, 0), (2, 4)],
[(i % 4, (i * 7) % block_size) for i in range(23)],
]
q_cpu = torch.randn(
len(selected_rows), num_heads, head_dim, generator=generator
).to(dtype)
cache_cpu = torch.randn(4, block_size, head_dim, generator=generator).to(dtype)
sinks_cpu = torch.linspace(-4.0, 6.0, num_heads)
# Layers can share a backing allocation, leaving unused rows between pages.
num_layers = 3 if interleaved_pages else 1
backing = torch.full(
(4, num_layers, block_size, head_dim),
float("nan"),
dtype=dtype,
device="cuda",
)
cache = backing[:, num_layers - 1]
cache.copy_(cache_cpu)
indices = [
page * num_layers * block_size + offset
for rows in selected_rows
for page, offset in rows
]
lengths = torch.tensor([0, *(len(rows) for rows in selected_rows)])
longest_row = max(len(rows) for rows in selected_rows)
metadata = SimpleNamespace(
block_size=block_size,
num_prefills=0,
num_decodes=len(selected_rows),
num_decode_tokens=len(selected_rows),
max_query_len=1,
max_seq_len=longest_row,
topk_tokens=longest_row,
qo_indptr=torch.arange(
len(selected_rows) + 1, dtype=torch.int32, device="cuda"
),
paged_kv_last_page_len=torch.ones(
len(selected_rows), dtype=torch.int32, device="cuda"
),
work_meta_data=None,
attn_out_dtype=dtype,
paged_kv_indices=torch.tensor(indices, dtype=torch.int32, device="cuda"),
paged_kv_indptr=lengths.cumsum(0).to(device="cuda", dtype=torch.int32),
)
impl = ROCMAiterMLASparseImpl.__new__(ROCMAiterMLASparseImpl)
impl.num_heads = num_heads
impl.head_size = head_dim
impl.kv_lora_rank = value_dim
impl.scale = head_dim**-0.5
impl.kv_cache_dtype = "auto"
impl.sinks = sinks_cpu.cuda()
padded_q = AiterMLAHelper.get_mla_padded_q(num_heads, q_cpu.cuda())
kv_rows, _ = flat_kv_row_view(cache, block_size)
actual, _ = impl._forward_mla(
SimpleNamespace(_q_scale=None, _k_scale=None),
padded_q,
kv_rows.unsqueeze(1),
metadata,
)
references = []
for query_idx, rows in enumerate(selected_rows):
keys = (
torch.stack([cache_cpu[page, offset] for page, offset in rows])
if rows
else cache_cpu.new_empty((0, head_dim))
)
references.append(
_sink_reference(q_cpu[query_idx], keys, sinks_cpu, impl.scale)
)
expected, _, value_scale, score_error = (
torch.stack(values) for values in zip(*references)
)
assert actual.shape == (len(selected_rows), num_heads, value_dim)
assert actual.dtype == dtype
_assert_sink_output_close(
actual,
expected,
value_scale,
score_error,
dtype,
native=dtype == torch.bfloat16,
)
@pytest.mark.parametrize("layout", ["LBNHC", "LBHNC", "BLNHC"])
def test_sparse_mla_backend_resolves_only_contiguous_layer_layouts(monkeypatch, layout):
from vllm.config import CacheConfig
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import (
ROCMAiterMLASparseBackend,
)
from vllm.v1.attention.backends.utils import resolve_kv_cache_layout
monkeypatch.setenv("VLLM_KV_CACHE_LAYOUT", layout)
config = SimpleNamespace(cache_config=CacheConfig())
supported = [
[x.name for x in ROCMAiterMLASparseBackend.supported_kv_cache_layouts()]
]
if layout == "BLNHC":
with pytest.raises(ValueError, match="does not satisfy"):
resolve_kv_cache_layout(config, supported)
else:
assert resolve_kv_cache_layout(config, supported).name == layout
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@torch.inference_mode()
def test_sparse_mla_sink_forward_mqa_preserves_split_query(dtype):
"""The public sparse forward joins latent/RoPE queries in the model dtype."""
_require_aiter()
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import (
ROCMAiterMLASparseImpl,
)
set_random_seed(412)
num_tokens, num_heads, block_size = 2, 8, 16
q = torch.randn(num_tokens, num_heads, Q_HEAD_DIM, device="cuda").to(dtype)
kv = torch.randn(2, 2, block_size, Q_HEAD_DIM, device="cuda").to(dtype)[:, 1]
selected = torch.tensor([[1, 7, 18], [0, 3, 20]], dtype=torch.int32, device="cuda")
sinks = torch.linspace(-3.0, 5.0, num_heads, device="cuda")
impl = object.__new__(ROCMAiterMLASparseImpl)
impl.num_heads = num_heads
impl.kv_lora_rank = V_HEAD_DIM
impl.kv_cache_dtype = "auto"
impl.scale = SM_SCALE
impl.sinks = sinks
impl.topk_indices_buffer = torch.full(
(num_tokens, 128), -1, dtype=torch.int32, device="cuda"
)
impl.topk_indices_buffer[:, : selected.shape[1]] = selected
impl.owns_indexer = True
impl.index_epoch = 0
impl.q_concat_buffer = torch.empty_like(q)
metadata = SimpleNamespace(
attn_out_dtype=dtype,
num_actual_tokens=num_tokens,
num_prefills=0,
num_decodes=num_tokens,
num_decode_tokens=num_tokens,
max_query_len=1,
max_seq_len=kv.shape[0],
block_size=block_size,
topk_tokens=impl.topk_indices_buffer.shape[1],
req_id_per_token=torch.zeros(num_tokens, dtype=torch.int32, device="cuda"),
block_table=torch.tensor([[0, 1]], dtype=torch.int32, device="cuda"),
qo_indptr=torch.arange(num_tokens + 1, dtype=torch.int32, device="cuda"),
paged_kv_indptr=torch.tensor([0, 3, 6], dtype=torch.int32, device="cuda"),
paged_kv_indices=torch.empty(
selected.numel(), dtype=torch.int32, device="cuda"
),
paged_kv_last_page_len=torch.ones(num_tokens, dtype=torch.int32, device="cuda"),
work_meta_data=None,
)
actual, _ = impl.forward_mqa(
(q[..., :V_HEAD_DIM], q[..., V_HEAD_DIM:]),
kv,
metadata,
SimpleNamespace(_q_scale=None, _k_scale=None),
)
references = [
_sink_reference(q[i], kv.flatten(0, 1)[selected[i].long()], sinks, SM_SCALE)
for i in range(num_tokens)
]
expected, _, value_scale, score_error = (
torch.stack(values) for values in zip(*references)
)
assert actual.dtype == dtype
_assert_sink_output_close(
actual,
expected,
value_scale,
score_error,
dtype,
native=dtype == torch.bfloat16,
)

Some files were not shown because too many files have changed in this diff Show More