[Mini-vLLM] SM86-only cut: models, backends, Rust, tests
- Keep only Qwen3.5, Qwen3.5-MTP and Qwen4Exp model paths; trim the model registry to 41 entries - Delete idle attention backends (MLA, FlashInfer, ROCm-only, quant backends) and dead GDN CuTeDSL kernels - Remove inc/quark quant loaders; keep compressed-tensors and Marlin - Drop the Rust frontend, pooling entrypoints, and idle test suites - Rebuild kernels for sm_80;sm_86 and strip the extensions - Verified: full import sweep clean (optional deps only) and a dummy-load Qwen3.5 engine run on RTX 3090 via the Triton GDN path Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,375 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import json
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.benchmarks.sweep import serve as sweep_serve
|
||||
from vllm.benchmarks.sweep.param_sweep import ParameterSweep, ParameterSweepItem
|
||||
|
||||
|
||||
class TestParameterSweepItem:
|
||||
"""Test ParameterSweepItem functionality."""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"input_dict,expected",
|
||||
[
|
||||
(
|
||||
{"compilation_config.use_inductor_graph_partition": False},
|
||||
"--compilation-config.use_inductor_graph_partition=false",
|
||||
),
|
||||
(
|
||||
{"compilation_config.use_inductor_graph_partition": True},
|
||||
"--compilation-config.use_inductor_graph_partition=true",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_nested_boolean_params(self, input_dict, expected):
|
||||
"""Test that nested boolean params use =true/false syntax."""
|
||||
item = ParameterSweepItem.from_record(input_dict)
|
||||
cmd = item.apply_to_cmd(["vllm", "serve", "model"])
|
||||
assert expected in cmd
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"input_dict,expected",
|
||||
[
|
||||
({"enable_prefix_caching": False}, "--no-enable-prefix-caching"),
|
||||
({"enable_prefix_caching": True}, "--enable-prefix-caching"),
|
||||
({"disable_log_stats": False}, "--no-disable-log-stats"),
|
||||
({"disable_log_stats": True}, "--disable-log-stats"),
|
||||
],
|
||||
)
|
||||
def test_non_nested_boolean_params(self, input_dict, expected):
|
||||
"""Test that non-nested boolean params use --no- prefix."""
|
||||
item = ParameterSweepItem.from_record(input_dict)
|
||||
cmd = item.apply_to_cmd(["vllm", "serve", "model"])
|
||||
assert expected in cmd
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"compilation_config",
|
||||
[
|
||||
{"cudagraph_mode": "full", "mode": 2, "use_inductor_graph_partition": True},
|
||||
{
|
||||
"cudagraph_mode": "piecewise",
|
||||
"mode": 3,
|
||||
"use_inductor_graph_partition": False,
|
||||
},
|
||||
],
|
||||
)
|
||||
def test_nested_dict_value(self, compilation_config):
|
||||
"""Test that nested dict values are serialized as JSON."""
|
||||
item = ParameterSweepItem.from_record(
|
||||
{"compilation_config": compilation_config}
|
||||
)
|
||||
cmd = item.apply_to_cmd(["vllm", "serve", "model"])
|
||||
assert "--compilation-config" in cmd
|
||||
# The dict should be JSON serialized
|
||||
idx = cmd.index("--compilation-config")
|
||||
assert json.loads(cmd[idx + 1]) == compilation_config
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"input_dict,expected_key,expected_value",
|
||||
[
|
||||
({"model": "test-model"}, "--model", "test-model"),
|
||||
({"max_tokens": 100}, "--max-tokens", "100"),
|
||||
({"temperature": 0.7}, "--temperature", "0.7"),
|
||||
],
|
||||
)
|
||||
def test_string_and_numeric_values(self, input_dict, expected_key, expected_value):
|
||||
"""Test that string and numeric values are handled correctly."""
|
||||
item = ParameterSweepItem.from_record(input_dict)
|
||||
cmd = item.apply_to_cmd(["vllm", "serve"])
|
||||
assert expected_key in cmd
|
||||
assert expected_value in cmd
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"input_dict,expected_key,key_idx_offset",
|
||||
[
|
||||
({"max_tokens": 200}, "--max-tokens", 1),
|
||||
({"enable_prefix_caching": False}, "--no-enable-prefix-caching", 0),
|
||||
],
|
||||
)
|
||||
def test_replace_existing_parameter(self, input_dict, expected_key, key_idx_offset):
|
||||
"""Test that existing parameters in cmd are replaced."""
|
||||
item = ParameterSweepItem.from_record(input_dict)
|
||||
|
||||
if key_idx_offset == 1:
|
||||
# Key-value pair
|
||||
cmd = item.apply_to_cmd(["vllm", "serve", "--max-tokens", "100", "model"])
|
||||
assert expected_key in cmd
|
||||
idx = cmd.index(expected_key)
|
||||
assert cmd[idx + 1] == "200"
|
||||
assert "100" not in cmd
|
||||
else:
|
||||
# Boolean flag
|
||||
cmd = item.apply_to_cmd(
|
||||
["vllm", "serve", "--enable-prefix-caching", "model"]
|
||||
)
|
||||
assert expected_key in cmd
|
||||
assert "--enable-prefix-caching" not in cmd
|
||||
|
||||
|
||||
class TestParameterSweep:
|
||||
"""Test ParameterSweep functionality."""
|
||||
|
||||
def test_from_records_list(self):
|
||||
"""Test creating ParameterSweep from a list of records."""
|
||||
records: list[dict[str, object]] = [
|
||||
{"max_tokens": 100, "temperature": 0.7},
|
||||
{"max_tokens": 200, "temperature": 0.9},
|
||||
]
|
||||
sweep = ParameterSweep.from_records(records)
|
||||
assert len(sweep) == 2
|
||||
assert sweep[0]["max_tokens"] == 100
|
||||
assert sweep[1]["max_tokens"] == 200
|
||||
|
||||
def test_read_from_dict(self):
|
||||
"""Test creating ParameterSweep from a dict format."""
|
||||
data: dict[str, dict[str, object]] = {
|
||||
"experiment1": {"max_tokens": 100, "temperature": 0.7},
|
||||
"experiment2": {"max_tokens": 200, "temperature": 0.9},
|
||||
}
|
||||
sweep = ParameterSweep.read_from_dict(data)
|
||||
assert len(sweep) == 2
|
||||
|
||||
# Check that items have the _benchmark_name field
|
||||
names = {item["_benchmark_name"] for item in sweep}
|
||||
assert names == {"experiment1", "experiment2"}
|
||||
|
||||
# Check that parameters are preserved
|
||||
for item in sweep:
|
||||
if item["_benchmark_name"] == "experiment1":
|
||||
assert item["max_tokens"] == 100
|
||||
assert item["temperature"] == 0.7
|
||||
elif item["_benchmark_name"] == "experiment2":
|
||||
assert item["max_tokens"] == 200
|
||||
assert item["temperature"] == 0.9
|
||||
|
||||
def test_read_json_list_format(self):
|
||||
"""Test reading JSON file with list format."""
|
||||
records: list[dict[str, object]] = [
|
||||
{"max_tokens": 100, "temperature": 0.7},
|
||||
{"max_tokens": 200, "temperature": 0.9},
|
||||
]
|
||||
|
||||
with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f:
|
||||
json.dump(records, f)
|
||||
temp_path = Path(f.name)
|
||||
|
||||
try:
|
||||
sweep = ParameterSweep.read_json(temp_path)
|
||||
assert len(sweep) == 2
|
||||
assert sweep[0]["max_tokens"] == 100
|
||||
assert sweep[1]["max_tokens"] == 200
|
||||
finally:
|
||||
temp_path.unlink()
|
||||
|
||||
def test_read_json_dict_format(self):
|
||||
"""Test reading JSON file with dict format."""
|
||||
data: dict[str, dict[str, object]] = {
|
||||
"experiment1": {"max_tokens": 100, "temperature": 0.7},
|
||||
"experiment2": {"max_tokens": 200, "temperature": 0.9},
|
||||
}
|
||||
|
||||
with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f:
|
||||
json.dump(data, f)
|
||||
temp_path = Path(f.name)
|
||||
|
||||
try:
|
||||
sweep = ParameterSweep.read_json(temp_path)
|
||||
assert len(sweep) == 2
|
||||
|
||||
# Check that items have the _benchmark_name field
|
||||
names = {item["_benchmark_name"] for item in sweep}
|
||||
assert names == {"experiment1", "experiment2"}
|
||||
finally:
|
||||
temp_path.unlink()
|
||||
|
||||
def test_unique_benchmark_names_validation(self):
|
||||
"""Test that duplicate _benchmark_name values raise an error."""
|
||||
# Test with duplicate names in list format
|
||||
records: list[dict[str, object]] = [
|
||||
{"_benchmark_name": "exp1", "max_tokens": 100},
|
||||
{"_benchmark_name": "exp1", "max_tokens": 200},
|
||||
]
|
||||
|
||||
with pytest.raises(ValueError, match="Duplicate _benchmark_name values"):
|
||||
ParameterSweep.from_records(records)
|
||||
|
||||
def test_unique_benchmark_names_multiple_duplicates(self):
|
||||
"""Test validation with multiple duplicate names."""
|
||||
records: list[dict[str, object]] = [
|
||||
{"_benchmark_name": "exp1", "max_tokens": 100},
|
||||
{"_benchmark_name": "exp1", "max_tokens": 200},
|
||||
{"_benchmark_name": "exp2", "max_tokens": 300},
|
||||
{"_benchmark_name": "exp2", "max_tokens": 400},
|
||||
]
|
||||
|
||||
with pytest.raises(ValueError, match="Duplicate _benchmark_name values"):
|
||||
ParameterSweep.from_records(records)
|
||||
|
||||
def test_no_benchmark_names_allowed(self):
|
||||
"""Test that records without _benchmark_name are allowed."""
|
||||
records: list[dict[str, object]] = [
|
||||
{"max_tokens": 100, "temperature": 0.7},
|
||||
{"max_tokens": 200, "temperature": 0.9},
|
||||
]
|
||||
sweep = ParameterSweep.from_records(records)
|
||||
assert len(sweep) == 2
|
||||
|
||||
def test_mixed_benchmark_names_allowed(self):
|
||||
"""Test that mixing records with and without _benchmark_name is allowed."""
|
||||
records: list[dict[str, object]] = [
|
||||
{"_benchmark_name": "exp1", "max_tokens": 100},
|
||||
{"max_tokens": 200, "temperature": 0.9},
|
||||
]
|
||||
sweep = ParameterSweep.from_records(records)
|
||||
assert len(sweep) == 2
|
||||
|
||||
|
||||
class TestParameterSweepItemKeyNormalization:
|
||||
"""Test key normalization in ParameterSweepItem."""
|
||||
|
||||
def test_underscore_to_hyphen_conversion(self):
|
||||
"""Test that underscores are converted to hyphens in CLI."""
|
||||
item = ParameterSweepItem.from_record({"max_tokens": 100})
|
||||
cmd = item.apply_to_cmd(["vllm", "serve"])
|
||||
assert "--max-tokens" in cmd
|
||||
|
||||
def test_nested_key_preserves_suffix(self):
|
||||
"""Test that nested keys preserve the suffix format."""
|
||||
# The suffix after the dot should preserve underscores
|
||||
item = ParameterSweepItem.from_record(
|
||||
{"compilation_config.some_nested_param": "value"}
|
||||
)
|
||||
cmd = item.apply_to_cmd(["vllm", "serve"])
|
||||
# The prefix (compilation_config) gets converted to hyphens,
|
||||
# but the suffix (some_nested_param) is preserved
|
||||
assert any("compilation-config.some_nested_param" in arg for arg in cmd)
|
||||
|
||||
|
||||
def test_run_comb_excludes_warmup_from_measured_results(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
):
|
||||
calls: list[tuple[int, int, str]] = []
|
||||
|
||||
def fake_run_benchmark(
|
||||
server,
|
||||
bench_cmd,
|
||||
*,
|
||||
serve_overrides,
|
||||
bench_overrides,
|
||||
run_number,
|
||||
output_path,
|
||||
dry_run,
|
||||
):
|
||||
calls.append(
|
||||
(run_number, int(bench_overrides["num_prompts"]), output_path.name)
|
||||
)
|
||||
return {"run_number": run_number}
|
||||
|
||||
monkeypatch.setattr(sweep_serve, "run_benchmark", fake_run_benchmark)
|
||||
base_path = tmp_path / "combination"
|
||||
base_path.mkdir()
|
||||
|
||||
measured = sweep_serve.run_comb(
|
||||
None,
|
||||
[],
|
||||
serve_comb=ParameterSweepItem(),
|
||||
bench_comb=ParameterSweepItem({"num_prompts": 320}),
|
||||
link_vars=[],
|
||||
base_path=base_path,
|
||||
num_runs=2,
|
||||
warmup_num_prompts=32,
|
||||
dry_run=False,
|
||||
)
|
||||
|
||||
assert calls == [
|
||||
(-1, 32, "warmup.json"),
|
||||
(0, 320, "run=0.json"),
|
||||
(1, 320, "run=1.json"),
|
||||
]
|
||||
assert measured == [{"run_number": 0}, {"run_number": 1}]
|
||||
|
||||
|
||||
def test_run_comb_continue_on_error_keeps_later_runs(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
):
|
||||
calls: list[int] = []
|
||||
|
||||
def fake_run_benchmark(
|
||||
server,
|
||||
bench_cmd,
|
||||
*,
|
||||
serve_overrides,
|
||||
bench_overrides,
|
||||
run_number,
|
||||
output_path,
|
||||
dry_run,
|
||||
):
|
||||
calls.append(run_number)
|
||||
if run_number == 0:
|
||||
raise RuntimeError("synthetic run failure")
|
||||
return {"run_number": run_number}
|
||||
|
||||
monkeypatch.setattr(sweep_serve, "run_benchmark", fake_run_benchmark)
|
||||
base_path = tmp_path / "combination"
|
||||
base_path.mkdir()
|
||||
|
||||
measured = sweep_serve.run_comb(
|
||||
None,
|
||||
[],
|
||||
serve_comb=ParameterSweepItem(),
|
||||
bench_comb=ParameterSweepItem({"num_prompts": 100}),
|
||||
link_vars=[],
|
||||
base_path=base_path,
|
||||
num_runs=2,
|
||||
warmup_num_prompts=0,
|
||||
dry_run=False,
|
||||
continue_on_error=True,
|
||||
)
|
||||
|
||||
assert calls == [0, 1]
|
||||
assert measured == [{"run_number": 1}]
|
||||
assert (base_path / "run=0.failure.json").exists()
|
||||
assert (base_path / "summary.json").exists()
|
||||
|
||||
|
||||
def test_run_comb_warmup_default_is_backward_compatible(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
):
|
||||
"""Older callers can omit warmup_num_prompts."""
|
||||
calls: list[int] = []
|
||||
|
||||
def fake_run_benchmark(
|
||||
server,
|
||||
bench_cmd,
|
||||
*,
|
||||
serve_overrides,
|
||||
bench_overrides,
|
||||
run_number,
|
||||
output_path,
|
||||
dry_run,
|
||||
):
|
||||
calls.append(run_number)
|
||||
return {"run_number": run_number}
|
||||
|
||||
monkeypatch.setattr(sweep_serve, "run_benchmark", fake_run_benchmark)
|
||||
base_path = tmp_path / "combination"
|
||||
base_path.mkdir()
|
||||
|
||||
measured = sweep_serve.run_comb(
|
||||
None,
|
||||
[],
|
||||
serve_comb=ParameterSweepItem(),
|
||||
bench_comb=ParameterSweepItem({"num_prompts": 10}),
|
||||
link_vars=[],
|
||||
base_path=base_path,
|
||||
num_runs=1,
|
||||
dry_run=False,
|
||||
)
|
||||
|
||||
assert calls == [0]
|
||||
assert measured == [{"run_number": 0}]
|
||||
@@ -1,330 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
from typing import Protocol, cast
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import soundfile as sf
|
||||
|
||||
import vllm.benchmarks.datasets.datasets as datasets_module
|
||||
import vllm.benchmarks.lib.endpoint_request_func as request_func_module
|
||||
from vllm.benchmarks.lib.endpoint_request_func import RequestFuncInput
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
|
||||
|
||||
class _ReadableBinary(Protocol):
|
||||
def read(self, size: int = -1) -> bytes: ...
|
||||
|
||||
|
||||
class _TokenizedPrompt:
|
||||
def __init__(self, prompt: str) -> None:
|
||||
self.input_ids = prompt.split()
|
||||
|
||||
|
||||
class _Tokenizer:
|
||||
def __init__(self, name_or_path: str = "openai/whisper-large-v3") -> None:
|
||||
self.name_or_path = name_or_path
|
||||
|
||||
def __call__(self, prompt: str) -> _TokenizedPrompt:
|
||||
return _TokenizedPrompt(prompt)
|
||||
|
||||
|
||||
class CohereAsrTokenizer(_Tokenizer):
|
||||
def __init__(self, name_or_path: str = "/models/cohere-transcribe") -> None:
|
||||
super().__init__(name_or_path)
|
||||
|
||||
|
||||
class _CohereNameOnlyTokenizer(_Tokenizer):
|
||||
def __init__(self) -> None:
|
||||
super().__init__("cohere/some-local-checkpoint")
|
||||
|
||||
|
||||
def _as_tokenizer(tokenizer: _Tokenizer) -> TokenizerLike:
|
||||
return cast(TokenizerLike, tokenizer)
|
||||
|
||||
|
||||
def _write_wav(path: Path, duration_s: float = 0.1, sample_rate: int = 16_000) -> None:
|
||||
num_samples = int(duration_s * sample_rate)
|
||||
sf.write(path, np.zeros(num_samples, dtype=np.float32), sample_rate)
|
||||
|
||||
|
||||
class _FakeFormData:
|
||||
def __init__(self) -> None:
|
||||
self.fields: list[tuple[str, object, dict[str, str]]] = []
|
||||
|
||||
def add_field(self, name: str, value: object, **kwargs: str) -> None:
|
||||
self.fields.append((name, value, kwargs))
|
||||
|
||||
|
||||
class _FakeContent:
|
||||
async def iter_any(self):
|
||||
yield b'data: {"choices":[{"delta":{"content":"hello"}}]}\n\n'
|
||||
yield b'data: {"usage":{"completion_tokens":1}}\n\n'
|
||||
yield b"data: [DONE]\n\n"
|
||||
|
||||
|
||||
class _FakeResponse:
|
||||
def __init__(self) -> None:
|
||||
self.status = 200
|
||||
self.reason = "OK"
|
||||
self.content = _FakeContent()
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
|
||||
class _FakeSession:
|
||||
def __init__(self) -> None:
|
||||
self.uploaded_bytes: bytes | None = None
|
||||
self.upload_filename: str | None = None
|
||||
self.fields: list[tuple[str, object, dict[str, str]]] | None = None
|
||||
|
||||
def post(self, *, url: str, data: _FakeFormData, headers: dict[str, str]):
|
||||
del url, headers
|
||||
self.fields = list(data.fields)
|
||||
_, file_obj, file_kwargs = self.fields[0]
|
||||
file_obj = cast(_ReadableBinary, file_obj)
|
||||
self.uploaded_bytes = file_obj.read()
|
||||
self.upload_filename = file_kwargs.get("filename")
|
||||
return _FakeResponse()
|
||||
|
||||
|
||||
class _FakeProfileSession:
|
||||
def __init__(self) -> None:
|
||||
self.url: str | None = None
|
||||
self.headers: dict[str, str] | None = None
|
||||
|
||||
def post(self, *, url: str, headers: dict[str, str]):
|
||||
self.url = url
|
||||
self.headers = headers
|
||||
return _FakeResponse()
|
||||
|
||||
|
||||
def test_asr_dataset_sample_handles_local_audio_paths(tmp_path: Path) -> None:
|
||||
audio_path = tmp_path / "earnings.wav"
|
||||
_write_wav(audio_path, duration_s=0.1)
|
||||
|
||||
dataset = object.__new__(datasets_module.ASRDataset)
|
||||
dataset.data = [
|
||||
{
|
||||
"audio": {
|
||||
"path": str(audio_path),
|
||||
"bytes": None,
|
||||
},
|
||||
"text": "quarterly earnings call",
|
||||
}
|
||||
]
|
||||
|
||||
samples = dataset.sample(
|
||||
tokenizer=_as_tokenizer(_Tokenizer()),
|
||||
num_requests=1,
|
||||
output_len=32,
|
||||
asr_min_audio_len_sec=0.0,
|
||||
asr_max_audio_len_sec=1.0,
|
||||
)
|
||||
|
||||
assert len(samples) == 1
|
||||
assert samples[0].multi_modal_data == {"audio_path": str(audio_path)}
|
||||
assert (
|
||||
samples[0].prompt == "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("has_filepath", [True, False])
|
||||
def test_asr_dataset_sample_handles_embedded_audio_bytes(
|
||||
tmp_path: Path, has_filepath: bool
|
||||
) -> None:
|
||||
audio_path = tmp_path / "earnings.wav"
|
||||
_write_wav(audio_path, duration_s=0.1)
|
||||
|
||||
test_path = None
|
||||
if has_filepath:
|
||||
test_path = audio_path
|
||||
|
||||
dataset = object.__new__(datasets_module.ASRDataset)
|
||||
dataset.data = [
|
||||
{
|
||||
"audio": {
|
||||
"path": test_path,
|
||||
"bytes": audio_path.read_bytes(),
|
||||
},
|
||||
"text": "quarterly earnings call",
|
||||
}
|
||||
]
|
||||
|
||||
samples = dataset.sample(
|
||||
tokenizer=_as_tokenizer(_Tokenizer()),
|
||||
num_requests=1,
|
||||
output_len=32,
|
||||
asr_min_audio_len_sec=0.0,
|
||||
asr_max_audio_len_sec=1.0,
|
||||
)
|
||||
|
||||
assert len(samples) == 1
|
||||
assert isinstance(samples[0].multi_modal_data, dict)
|
||||
audio, sample_rate = samples[0].multi_modal_data["audio"]
|
||||
assert sample_rate == 16_000
|
||||
assert isinstance(audio, np.ndarray)
|
||||
assert audio.size > 0
|
||||
|
||||
|
||||
def test_async_request_openai_audio_handles_local_audio_paths(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
audio_path = tmp_path / "earnings.wav"
|
||||
_write_wav(audio_path, duration_s=0.25)
|
||||
|
||||
monkeypatch.setattr(request_func_module.aiohttp, "FormData", _FakeFormData)
|
||||
session = _FakeSession()
|
||||
request_input = RequestFuncInput(
|
||||
prompt="",
|
||||
api_url="http://localhost:8000/v1/audio/transcriptions",
|
||||
prompt_len=1,
|
||||
output_len=32,
|
||||
model="openai/whisper-large-v3",
|
||||
multi_modal_content={"audio_path": str(audio_path)},
|
||||
)
|
||||
|
||||
output = asyncio.run(
|
||||
request_func_module.async_request_openai_audio(request_input, session)
|
||||
)
|
||||
|
||||
assert session.upload_filename == audio_path.name
|
||||
assert session.uploaded_bytes == audio_path.read_bytes()
|
||||
assert output.success is True
|
||||
assert output.generated_text == "hello"
|
||||
assert output.output_tokens == 1
|
||||
assert output.input_audio_duration == pytest.approx(0.25, abs=1e-2)
|
||||
|
||||
|
||||
def test_async_request_openai_audio_handles_decoded_audio_arrays(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(request_func_module.aiohttp, "FormData", _FakeFormData)
|
||||
session = _FakeSession()
|
||||
request_input = RequestFuncInput(
|
||||
prompt="",
|
||||
api_url="http://localhost:8000/v1/audio/transcriptions",
|
||||
prompt_len=1,
|
||||
output_len=32,
|
||||
model="openai/whisper-large-v3",
|
||||
multi_modal_content={
|
||||
"audio": (np.zeros(1_600, dtype=np.float32), 16_000),
|
||||
},
|
||||
)
|
||||
|
||||
output = asyncio.run(
|
||||
request_func_module.async_request_openai_audio(request_input, session)
|
||||
)
|
||||
|
||||
assert session.upload_filename == "audio.wav"
|
||||
assert session.uploaded_bytes is not None
|
||||
assert output.success is True
|
||||
assert output.generated_text == "hello"
|
||||
|
||||
|
||||
def test_audio_request_rejects_profile_endpoint() -> None:
|
||||
request_input = RequestFuncInput(
|
||||
prompt="",
|
||||
api_url="http://localhost:8000/start_profile",
|
||||
prompt_len=1,
|
||||
output_len=32,
|
||||
model="openai/whisper-large-v3",
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="OpenAI Audio API"):
|
||||
asyncio.run(
|
||||
request_func_module.async_request_openai_audio(
|
||||
request_input, _FakeProfileSession()
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("endpoint", ["start_profile", "stop_profile"])
|
||||
def test_audio_benchmark_profile_request_has_no_inference_payload(
|
||||
endpoint: str,
|
||||
) -> None:
|
||||
session = _FakeProfileSession()
|
||||
|
||||
output = asyncio.run(
|
||||
request_func_module.async_request_profile(
|
||||
f"http://localhost:8000/{endpoint}",
|
||||
session,
|
||||
extra_headers={"X-Test-Header": "profile"},
|
||||
)
|
||||
)
|
||||
|
||||
assert output.success is True
|
||||
assert session.url == f"http://localhost:8000/{endpoint}"
|
||||
assert session.headers == {"X-Test-Header": "profile"}
|
||||
|
||||
|
||||
_COHERE_ASR_PROMPT = (
|
||||
"<|startofcontext|><|startoftranscript|>"
|
||||
"<|emo:undefined|><|en|><|en|><|pnc|><|noitn|>"
|
||||
"<|notimestamp|><|nodiarize|>"
|
||||
)
|
||||
|
||||
|
||||
def _make_asr_dataset(tmp_path: Path) -> datasets_module.ASRDataset:
|
||||
audio_path = tmp_path / "sample.wav"
|
||||
_write_wav(audio_path, duration_s=0.1)
|
||||
dataset = object.__new__(datasets_module.ASRDataset)
|
||||
dataset.data = [
|
||||
{
|
||||
"audio": {"path": str(audio_path), "bytes": None},
|
||||
"text": "hello world",
|
||||
}
|
||||
]
|
||||
return dataset
|
||||
|
||||
|
||||
def test_asr_dataset_cohere_class_name_gets_decoder_prompt(tmp_path: Path) -> None:
|
||||
dataset = _make_asr_dataset(tmp_path)
|
||||
samples = dataset.sample(
|
||||
tokenizer=_as_tokenizer(CohereAsrTokenizer()),
|
||||
num_requests=1,
|
||||
output_len=32,
|
||||
asr_min_audio_len_sec=0.0,
|
||||
asr_max_audio_len_sec=1.0,
|
||||
)
|
||||
assert len(samples) == 1
|
||||
assert samples[0].prompt == _COHERE_ASR_PROMPT
|
||||
|
||||
|
||||
def test_asr_dataset_cohere_name_or_path_fallback_gets_decoder_prompt(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
dataset = _make_asr_dataset(tmp_path)
|
||||
samples = dataset.sample(
|
||||
tokenizer=_as_tokenizer(_CohereNameOnlyTokenizer()),
|
||||
num_requests=1,
|
||||
output_len=32,
|
||||
asr_min_audio_len_sec=0.0,
|
||||
asr_max_audio_len_sec=1.0,
|
||||
)
|
||||
assert len(samples) == 1
|
||||
assert samples[0].prompt == _COHERE_ASR_PROMPT
|
||||
|
||||
|
||||
def test_asr_dataset_unknown_tokenizer_gets_empty_prompt(tmp_path: Path) -> None:
|
||||
dataset = _make_asr_dataset(tmp_path)
|
||||
samples = dataset.sample(
|
||||
tokenizer=_as_tokenizer(_Tokenizer(name_or_path="some-other/asr-model")),
|
||||
num_requests=1,
|
||||
output_len=32,
|
||||
asr_min_audio_len_sec=0.0,
|
||||
asr_max_audio_len_sec=1.0,
|
||||
)
|
||||
assert len(samples) == 1
|
||||
assert samples[0].prompt == ""
|
||||
@@ -1,341 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from transformers import AutoTokenizer, PreTrainedTokenizerBase
|
||||
|
||||
from vllm.benchmarks.datasets import BFCLDataset, get_samples
|
||||
|
||||
|
||||
def _patch_hf_api(side_effect):
|
||||
"""Return a patch context that swaps `hf_api()` to a stub whose
|
||||
`.hf_hub_download` attribute uses `side_effect`."""
|
||||
fake_api = MagicMock()
|
||||
fake_api.hf_hub_download.side_effect = side_effect
|
||||
return patch("vllm.benchmarks.datasets.datasets.hf_api", return_value=fake_api)
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def hf_tokenizer() -> PreTrainedTokenizerBase:
|
||||
return AutoTokenizer.from_pretrained("openai-community/gpt2")
|
||||
|
||||
|
||||
_FAKE_ROWS = {
|
||||
"simple": [
|
||||
{
|
||||
"id": "simple_0",
|
||||
"question": [
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "What is 2+2?",
|
||||
}
|
||||
]
|
||||
],
|
||||
"function": [
|
||||
{
|
||||
"name": "add",
|
||||
"description": "Add two numbers.",
|
||||
"parameters": {
|
||||
"type": "dict",
|
||||
"properties": {
|
||||
"a": {"type": "integer", "description": "first"},
|
||||
"b": {"type": "float", "description": "second"},
|
||||
},
|
||||
"required": ["a", "b"],
|
||||
},
|
||||
}
|
||||
],
|
||||
},
|
||||
],
|
||||
"live_simple": [
|
||||
{
|
||||
"id": "live_simple_0",
|
||||
"question": [[{"role": "user", "content": "Tell me the weather."}]],
|
||||
"function": [
|
||||
{
|
||||
"name": "get_weather",
|
||||
"description": "Get weather.",
|
||||
"parameters": {
|
||||
"type": "dict",
|
||||
"properties": {
|
||||
"city": {"type": "any", "description": "city"},
|
||||
"coords": {"type": "tuple", "description": "coords"},
|
||||
},
|
||||
"required": ["city"],
|
||||
},
|
||||
}
|
||||
],
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _write_fake_files(tmp_path: Path) -> dict[str, Path]:
|
||||
"""Write fake BFCL JSONL files mimicking the HF repo layout."""
|
||||
paths = {}
|
||||
for category, rows in _FAKE_ROWS.items():
|
||||
p = tmp_path / f"BFCL_v3_{category}.json"
|
||||
with p.open("w") as f:
|
||||
for row in rows:
|
||||
f.write(json.dumps(row) + "\n")
|
||||
paths[category] = p
|
||||
return paths
|
||||
|
||||
|
||||
def _args_for_bfcl(categories: list[str] | None) -> argparse.Namespace:
|
||||
return argparse.Namespace(
|
||||
dataset_name="hf",
|
||||
dataset_path="gorilla-llm/Berkeley-Function-Calling-Leaderboard",
|
||||
hf_name=None,
|
||||
hf_subset=None,
|
||||
hf_split=None,
|
||||
hf_output_len=64,
|
||||
disable_shuffle=True,
|
||||
num_prompts=2,
|
||||
no_oversample=False,
|
||||
no_stream=True,
|
||||
seed=0,
|
||||
request_id_prefix="",
|
||||
trust_remote_code=False,
|
||||
skip_chat_template=False,
|
||||
enable_multimodal_chat=False,
|
||||
backend="openai-chat",
|
||||
bfcl_categories=categories,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_bfcl_dataset_translates_schema_and_attaches_tools(
|
||||
hf_tokenizer: PreTrainedTokenizerBase, tmp_path: Path
|
||||
) -> None:
|
||||
"""BFCLDataset should translate schemas to OpenAI tool format, set
|
||||
`messages` directly on SampleRequest, and attach tools/tool_choice via
|
||||
request_overrides."""
|
||||
paths = _write_fake_files(tmp_path)
|
||||
|
||||
def fake_download(_repo, filename, **_kwargs):
|
||||
category = filename.removeprefix("BFCL_v3_").removesuffix(".json")
|
||||
return str(paths[category])
|
||||
|
||||
args = _args_for_bfcl(categories=["simple", "live_simple"])
|
||||
|
||||
with _patch_hf_api(fake_download):
|
||||
samples = get_samples(args, hf_tokenizer)
|
||||
|
||||
assert len(samples) == 2
|
||||
for s in samples:
|
||||
assert s.chat_messages is not None
|
||||
assert isinstance(s.chat_messages, list)
|
||||
assert s.chat_messages[0]["role"] == "user"
|
||||
assert s.request_overrides is not None
|
||||
assert "tools" in s.request_overrides
|
||||
assert s.request_overrides["tool_choice"] == "auto"
|
||||
# messages must NOT leak into request_overrides — it has its own
|
||||
# typed field on SampleRequest.
|
||||
assert "messages" not in s.request_overrides
|
||||
tools = s.request_overrides["tools"]
|
||||
assert len(tools) == 1
|
||||
tool = tools[0]
|
||||
assert tool["type"] == "function"
|
||||
# Translated schema: dict -> object, float -> number,
|
||||
# any -> string, tuple -> array.
|
||||
params = tool["function"]["parameters"]
|
||||
assert params["type"] == "object"
|
||||
for prop in params["properties"].values():
|
||||
assert prop["type"] in {"integer", "number", "string", "array"}
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_bfcl_dataset_requires_openai_chat_backend(
|
||||
hf_tokenizer: PreTrainedTokenizerBase,
|
||||
) -> None:
|
||||
args = _args_for_bfcl(categories=["simple"])
|
||||
args.backend = "openai"
|
||||
|
||||
with pytest.raises(ValueError, match="openai-chat"):
|
||||
get_samples(args, hf_tokenizer)
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_bfcl_dataset_missing_category_raises_clear_error(
|
||||
hf_tokenizer: PreTrainedTokenizerBase,
|
||||
) -> None:
|
||||
"""A typo'd category should produce an actionable ValueError, not an
|
||||
opaque huggingface_hub exception."""
|
||||
from huggingface_hub.errors import EntryNotFoundError
|
||||
|
||||
args = _args_for_bfcl(categories=["simpl"]) # typo
|
||||
|
||||
def raise_missing(_repo, filename, **_kwargs):
|
||||
raise EntryNotFoundError(f"404 Not Found: {filename}")
|
||||
|
||||
with (
|
||||
_patch_hf_api(raise_missing),
|
||||
pytest.raises(ValueError, match=r"BFCL category 'simpl' not found"),
|
||||
):
|
||||
get_samples(args, hf_tokenizer)
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_chat_backend_uses_messages_field_when_set() -> None:
|
||||
"""When RequestFuncInput.chat_messages is set, the chat backend must use
|
||||
it verbatim and skip default content construction from `prompt`."""
|
||||
import asyncio
|
||||
|
||||
from vllm.benchmarks.lib.endpoint_request_func import (
|
||||
RequestFuncInput,
|
||||
async_request_openai_chat_completions,
|
||||
)
|
||||
|
||||
captured: dict = {}
|
||||
|
||||
class _FakeResp:
|
||||
status = 500
|
||||
reason = "stop-after-capture"
|
||||
content = None
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *exc):
|
||||
return False
|
||||
|
||||
class _FakeSession:
|
||||
def post(self, url, json, headers): # noqa: A002
|
||||
captured["url"] = url
|
||||
captured["payload"] = json
|
||||
return _FakeResp()
|
||||
|
||||
messages = [
|
||||
{"role": "system", "content": "you are helpful"},
|
||||
{"role": "user", "content": "call add(3, 4)"},
|
||||
]
|
||||
req = RequestFuncInput(
|
||||
prompt="IGNORED",
|
||||
api_url="http://localhost:0/v1/chat/completions",
|
||||
prompt_len=10,
|
||||
output_len=16,
|
||||
model="test-model",
|
||||
chat_messages=messages,
|
||||
extra_body={"tools": [{"type": "function", "function": {"name": "add"}}]},
|
||||
)
|
||||
|
||||
asyncio.run(
|
||||
async_request_openai_chat_completions(
|
||||
request_func_input=req, session=_FakeSession()
|
||||
)
|
||||
)
|
||||
|
||||
payload = captured["payload"]
|
||||
assert payload["messages"] is messages, (
|
||||
"chat backend must forward RequestFuncInput.chat_messages verbatim "
|
||||
"instead of constructing a default user message from `prompt`"
|
||||
)
|
||||
# extra_body still merges in as before (shallow, per-request wins).
|
||||
assert payload["tools"][0]["function"]["name"] == "add"
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_bfcl_prompt_len_includes_tools(tmp_path: Path) -> None:
|
||||
"""prompt_len must reflect tokens from both messages *and* tool schemas,
|
||||
so percentile buckets and input-distribution summaries aren't biased
|
||||
low for tool-heavy traffic."""
|
||||
paths = _write_fake_files(tmp_path)
|
||||
|
||||
def fake_download(_repo, filename, **_kwargs):
|
||||
category = filename.removeprefix("BFCL_v3_").removesuffix(".json")
|
||||
return str(paths[category])
|
||||
|
||||
captured: dict = {}
|
||||
|
||||
class _FakeTokenizer:
|
||||
def apply_chat_template(
|
||||
self, messages, tools=None, tokenize=False, add_generation_prompt=True
|
||||
):
|
||||
captured["tools"] = tools
|
||||
base = " ".join(m.get("content", "") for m in messages)
|
||||
tool_text = json.dumps(tools) if tools else ""
|
||||
return base + " " + tool_text
|
||||
|
||||
def __call__(self, text):
|
||||
# 1 "token" per whitespace-separated word.
|
||||
return type("Enc", (), {"input_ids": text.split()})()
|
||||
|
||||
fake = _FakeTokenizer()
|
||||
args = _args_for_bfcl(categories=["simple"])
|
||||
args.num_prompts = 1
|
||||
|
||||
with _patch_hf_api(fake_download):
|
||||
samples = get_samples(args, fake)
|
||||
|
||||
assert len(samples) == 1
|
||||
assert captured["tools"] is not None, (
|
||||
"apply_chat_template must be called with tools= so the schema "
|
||||
"contributes to the prompt-length estimate"
|
||||
)
|
||||
assert len(captured["tools"]) == 1
|
||||
assert captured["tools"][0]["function"]["name"] == "add"
|
||||
|
||||
# Sanity: prompt_len exceeds a messages-only estimate. The fake row's
|
||||
# user message is "What is 2+2?" (3 whitespace-separated tokens).
|
||||
assert samples[0].prompt_len > 3
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_bfcl_prompt_len_falls_back_when_tokenizer_rejects_tools(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""Older tokenizers don't accept tools=; fallback must still produce a
|
||||
non-zero prompt_len without crashing."""
|
||||
paths = _write_fake_files(tmp_path)
|
||||
|
||||
def fake_download(_repo, filename, **_kwargs):
|
||||
category = filename.removeprefix("BFCL_v3_").removesuffix(".json")
|
||||
return str(paths[category])
|
||||
|
||||
class _LegacyTokenizer:
|
||||
def apply_chat_template(self, messages, **kwargs):
|
||||
if "tools" in kwargs:
|
||||
raise TypeError("unexpected keyword argument 'tools'")
|
||||
return " ".join(m.get("content", "") for m in messages)
|
||||
|
||||
def __call__(self, text):
|
||||
return type("Enc", (), {"input_ids": text.split()})()
|
||||
|
||||
args = _args_for_bfcl(categories=["simple"])
|
||||
args.num_prompts = 1
|
||||
|
||||
with _patch_hf_api(fake_download):
|
||||
samples = get_samples(args, _LegacyTokenizer())
|
||||
|
||||
assert len(samples) == 1
|
||||
assert samples[0].prompt_len > 0
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_bfcl_schema_translation_is_recursive() -> None:
|
||||
"""_translate_schema must recurse into nested properties."""
|
||||
input_schema = {
|
||||
"type": "dict",
|
||||
"properties": {
|
||||
"nested": {
|
||||
"type": "dict",
|
||||
"properties": {
|
||||
"value": {"type": "float"},
|
||||
"tags": {"type": "tuple", "items": {"type": "any"}},
|
||||
},
|
||||
}
|
||||
},
|
||||
}
|
||||
out = BFCLDataset._translate_schema(input_schema)
|
||||
assert out["type"] == "object"
|
||||
assert out["properties"]["nested"]["type"] == "object"
|
||||
assert out["properties"]["nested"]["properties"]["value"]["type"] == "number"
|
||||
assert out["properties"]["nested"]["properties"]["tags"]["type"] == "array"
|
||||
nested_props = out["properties"]["nested"]["properties"]
|
||||
assert nested_props["tags"]["items"]["type"] == "string"
|
||||
@@ -1,75 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.benchmarks.datasets import get_samples
|
||||
|
||||
|
||||
class _RecordingTokenizer:
|
||||
"""Minimal tokenizer stub that records the kwargs forwarded to
|
||||
apply_chat_template, so we can assert chat_template_kwargs propagation
|
||||
without loading a real model/template."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.captured_kwargs: dict | None = None
|
||||
self.chat_template = "dummy-template"
|
||||
|
||||
def apply_chat_template(
|
||||
self,
|
||||
conversation,
|
||||
add_generation_prompt: bool = True,
|
||||
tokenize: bool = False,
|
||||
**kwargs,
|
||||
) -> str:
|
||||
self.captured_kwargs = kwargs
|
||||
return conversation[0]["content"]
|
||||
|
||||
def __call__(self, text: str):
|
||||
return argparse.Namespace(input_ids=list(range(len(text.split()))))
|
||||
|
||||
|
||||
def _args(dataset_path: str, chat_template_kwargs) -> argparse.Namespace:
|
||||
return argparse.Namespace(
|
||||
dataset_name="custom",
|
||||
dataset_path=dataset_path,
|
||||
disable_shuffle=True,
|
||||
num_prompts=1,
|
||||
custom_output_len=32,
|
||||
skip_chat_template=False,
|
||||
chat_template_kwargs=chat_template_kwargs,
|
||||
no_oversample=False,
|
||||
seed=0,
|
||||
request_id_prefix="",
|
||||
)
|
||||
|
||||
|
||||
def _write_one(path: Path) -> None:
|
||||
path.write_text(json.dumps({"prompt": "hello world"}) + "\n")
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_chat_template_kwargs_forwarded(tmp_path: Path) -> None:
|
||||
"""--chat-template-kwargs must reach the client-side apply_chat_template."""
|
||||
jsonl = tmp_path / "data.jsonl"
|
||||
_write_one(jsonl)
|
||||
|
||||
tok = _RecordingTokenizer()
|
||||
get_samples(_args(str(jsonl), {"thinking": True}), tok)
|
||||
|
||||
assert tok.captured_kwargs == {"thinking": True}
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_chat_template_kwargs_default_is_noop(tmp_path: Path) -> None:
|
||||
"""When not provided, no extra kwargs are passed (existing behavior)."""
|
||||
jsonl = tmp_path / "data.jsonl"
|
||||
_write_one(jsonl)
|
||||
|
||||
tok = _RecordingTokenizer()
|
||||
get_samples(_args(str(jsonl), None), tok)
|
||||
|
||||
assert tok.captured_kwargs == {}
|
||||
@@ -1,77 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from transformers import AutoTokenizer, PreTrainedTokenizerBase
|
||||
|
||||
from vllm.benchmarks.datasets import get_samples
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def hf_tokenizer() -> PreTrainedTokenizerBase:
|
||||
return AutoTokenizer.from_pretrained("openai-community/gpt2")
|
||||
|
||||
|
||||
def _write_jsonl(path: Path, n_rows: int) -> None:
|
||||
with path.open("w") as f:
|
||||
for i in range(n_rows):
|
||||
f.write(json.dumps({"prompt": f"row {i}: unique prompt content."}) + "\n")
|
||||
|
||||
|
||||
def _args_for_custom(dataset_path: str, seed: int) -> argparse.Namespace:
|
||||
return argparse.Namespace(
|
||||
dataset_name="custom",
|
||||
dataset_path=dataset_path,
|
||||
disable_shuffle=False,
|
||||
num_prompts=30,
|
||||
custom_output_len=32,
|
||||
skip_chat_template=True,
|
||||
no_oversample=False,
|
||||
seed=seed,
|
||||
request_id_prefix="",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_custom_dataset_seed_propagates(
|
||||
hf_tokenizer: PreTrainedTokenizerBase, tmp_path: Path
|
||||
) -> None:
|
||||
"""--seed must control the CustomDataset shuffle used by get_samples.
|
||||
|
||||
Without the fix, CustomDataset was instantiated without random_seed,
|
||||
so its load-time shuffle always used DEFAULT_SEED=0 regardless of
|
||||
args.seed, causing every run with --dataset-name custom to pick the
|
||||
same subset of rows from a larger file.
|
||||
"""
|
||||
jsonl = tmp_path / "data.jsonl"
|
||||
_write_jsonl(jsonl, n_rows=60)
|
||||
|
||||
samples_a = get_samples(_args_for_custom(str(jsonl), seed=0), hf_tokenizer)
|
||||
samples_b = get_samples(_args_for_custom(str(jsonl), seed=42), hf_tokenizer)
|
||||
|
||||
prompts_a = {s.prompt for s in samples_a}
|
||||
prompts_b = {s.prompt for s in samples_b}
|
||||
|
||||
assert len(prompts_a) == 30
|
||||
assert len(prompts_b) == 30
|
||||
assert prompts_a != prompts_b
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_custom_dataset_same_seed_is_deterministic(
|
||||
hf_tokenizer: PreTrainedTokenizerBase, tmp_path: Path
|
||||
) -> None:
|
||||
"""Same --seed must yield the same CustomDataset subset."""
|
||||
jsonl = tmp_path / "data.jsonl"
|
||||
_write_jsonl(jsonl, n_rows=60)
|
||||
|
||||
samples_a = get_samples(_args_for_custom(str(jsonl), seed=7), hf_tokenizer)
|
||||
samples_b = get_samples(_args_for_custom(str(jsonl), seed=7), hf_tokenizer)
|
||||
|
||||
prompts_a = [s.prompt for s in samples_a]
|
||||
prompts_b = [s.prompt for s in samples_b]
|
||||
|
||||
assert prompts_a == prompts_b
|
||||
@@ -1,400 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import json
|
||||
from argparse import Namespace
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
from typing import Any, cast
|
||||
|
||||
import pybase64 as base64
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
import vllm.benchmarks.datasets.datasets as datasets_module
|
||||
from vllm.benchmarks.datasets import CustomImageDataset, get_samples
|
||||
from vllm.benchmarks.lib.endpoint_request_func import (
|
||||
RequestFuncInput,
|
||||
_get_chat_content,
|
||||
_get_chat_messages,
|
||||
)
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
|
||||
|
||||
class _TokenizedPrompt:
|
||||
def __init__(self, prompt: str) -> None:
|
||||
self.input_ids = prompt.split()
|
||||
|
||||
|
||||
class _Tokenizer:
|
||||
def __call__(self, prompt: str) -> _TokenizedPrompt:
|
||||
return _TokenizedPrompt(prompt)
|
||||
|
||||
|
||||
def _tokenizer() -> TokenizerLike:
|
||||
return cast(TokenizerLike, _Tokenizer())
|
||||
|
||||
|
||||
def _write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None:
|
||||
with path.open("w") as f:
|
||||
for row in rows:
|
||||
f.write(json.dumps(row) + "\n")
|
||||
|
||||
|
||||
def _write_png(path: Path, color: tuple[int, int, int] = (255, 0, 0)) -> None:
|
||||
Image.new("RGB", (1, 1), color=color).save(path)
|
||||
|
||||
|
||||
def _decode_data_url(data_url: str) -> tuple[str, bytes]:
|
||||
prefix, image_base64 = data_url.split(",", 1)
|
||||
return prefix, base64.b64decode(image_base64)
|
||||
|
||||
|
||||
def _assert_png_data_url(data_url: str) -> None:
|
||||
prefix, image_bytes = _decode_data_url(data_url)
|
||||
assert prefix == "data:image/png;base64"
|
||||
with Image.open(BytesIO(image_bytes)) as image:
|
||||
image.verify()
|
||||
|
||||
|
||||
def _args_for_custom_image(dataset_path: Path) -> Namespace:
|
||||
return Namespace(
|
||||
dataset_name="custom_image",
|
||||
dataset_path=str(dataset_path),
|
||||
disable_shuffle=True,
|
||||
seed=0,
|
||||
num_prompts=2,
|
||||
custom_output_len=32,
|
||||
enable_multimodal_chat=False,
|
||||
custom_ensure_client_side_data=False,
|
||||
request_id_prefix="req-",
|
||||
no_oversample=False,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_get_samples_custom_image_cli_path_supports_multi_image_and_content(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
image_a = tmp_path / "chart_a.png"
|
||||
image_b = tmp_path / "chart_b.png"
|
||||
image_c = tmp_path / "chart_c.png"
|
||||
jsonl = tmp_path / "images.jsonl"
|
||||
_write_jsonl(
|
||||
jsonl,
|
||||
[
|
||||
{
|
||||
"prompt": "Compare the first two charts.",
|
||||
"image_files": [str(image_a), str(image_b)],
|
||||
},
|
||||
{
|
||||
"content": [
|
||||
{"type": "text", "text": "Now compare "},
|
||||
{"type": "image", "image": str(image_c)},
|
||||
],
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
samples = get_samples(_args_for_custom_image(jsonl), _tokenizer())
|
||||
|
||||
assert len(samples) == 2
|
||||
assert samples[0].request_id == "req-0"
|
||||
assert isinstance(samples[0].multi_modal_data, list)
|
||||
assert [part["image_url"]["url"] for part in samples[0].multi_modal_data] == [
|
||||
f"file://{image_a}",
|
||||
f"file://{image_b}",
|
||||
]
|
||||
|
||||
assert samples[1].request_id == "req-1"
|
||||
assert samples[1].multi_modal_data is None
|
||||
assert isinstance(samples[1].prompt, list)
|
||||
assert samples[1].prompt[0] == {"type": "text", "text": "Now compare "}
|
||||
assert isinstance(samples[1].prompt[1], dict)
|
||||
assert samples[1].prompt[1]["image_url"]["url"] == f"file://{image_c}"
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_custom_image_dataset_uses_all_image_files(tmp_path: Path) -> None:
|
||||
image_a = tmp_path / "chart_a.png"
|
||||
image_b = tmp_path / "chart_b.png"
|
||||
jsonl = tmp_path / "images.jsonl"
|
||||
_write_jsonl(
|
||||
jsonl,
|
||||
[
|
||||
{
|
||||
"prompt": "Compare the charts.",
|
||||
"image_files": [str(image_a), str(image_b)],
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
|
||||
samples = dataset.sample(
|
||||
tokenizer=_tokenizer(),
|
||||
num_requests=1,
|
||||
output_len=32,
|
||||
)
|
||||
|
||||
assert len(samples) == 1
|
||||
sample = samples[0]
|
||||
assert sample.prompt == "Compare the charts."
|
||||
assert sample.prompt_len == 3
|
||||
assert isinstance(sample.multi_modal_data, list)
|
||||
assert [part["image_url"]["url"] for part in sample.multi_modal_data] == [
|
||||
f"file://{image_a}",
|
||||
f"file://{image_b}",
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_custom_image_dataset_preserves_interleaved_content_order(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
image_a = tmp_path / "chart_a.png"
|
||||
image_b = tmp_path / "chart_b.png"
|
||||
jsonl = tmp_path / "images.jsonl"
|
||||
_write_jsonl(
|
||||
jsonl,
|
||||
[
|
||||
{
|
||||
"content": [
|
||||
{"type": "text", "text": "Compare "},
|
||||
{"type": "image", "image": str(image_a)},
|
||||
{"type": "text", "text": " with "},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": str(image_b),
|
||||
"detail": "low",
|
||||
},
|
||||
},
|
||||
],
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
|
||||
samples = dataset.sample(
|
||||
tokenizer=_tokenizer(),
|
||||
num_requests=1,
|
||||
output_len=32,
|
||||
)
|
||||
|
||||
assert len(samples) == 1
|
||||
sample = samples[0]
|
||||
assert sample.multi_modal_data is None
|
||||
assert sample.prompt_len == 2
|
||||
assert isinstance(sample.prompt, list)
|
||||
parts = []
|
||||
for part in sample.prompt:
|
||||
assert isinstance(part, dict)
|
||||
parts.append(part)
|
||||
assert [part["type"] for part in parts] == [
|
||||
"text",
|
||||
"image_url",
|
||||
"text",
|
||||
"image_url",
|
||||
]
|
||||
assert parts[1]["image_url"]["url"] == f"file://{image_a}"
|
||||
assert parts[3]["image_url"] == {
|
||||
"url": f"file://{image_b}",
|
||||
"detail": "low",
|
||||
}
|
||||
|
||||
request_input = RequestFuncInput(
|
||||
prompt=sample.prompt,
|
||||
api_url="http://localhost:8000/v1/chat/completions",
|
||||
prompt_len=sample.prompt_len,
|
||||
output_len=32,
|
||||
model="test-model",
|
||||
)
|
||||
assert _get_chat_content(request_input) == sample.prompt
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_custom_image_dataset_wraps_interleaved_content_for_multimodal_chat(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
image = tmp_path / "chart.png"
|
||||
jsonl = tmp_path / "images.jsonl"
|
||||
_write_jsonl(
|
||||
jsonl,
|
||||
[
|
||||
{
|
||||
"content": [
|
||||
{"type": "text", "text": "Describe "},
|
||||
{"type": "image", "image": str(image)},
|
||||
],
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
|
||||
samples = dataset.sample(
|
||||
tokenizer=_tokenizer(),
|
||||
num_requests=1,
|
||||
output_len=32,
|
||||
enable_multimodal_chat=True,
|
||||
)
|
||||
|
||||
sample = samples[0]
|
||||
assert sample.multi_modal_data is None
|
||||
assert sample.prompt == [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Describe "},
|
||||
{"type": "image_url", "image_url": {"url": f"file://{image}"}},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
request_input = RequestFuncInput(
|
||||
prompt=sample.prompt,
|
||||
api_url="http://localhost:8000/v1/chat/completions",
|
||||
prompt_len=sample.prompt_len,
|
||||
output_len=32,
|
||||
model="test-model",
|
||||
)
|
||||
assert _get_chat_messages(request_input) == sample.prompt
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_custom_image_dataset_encodes_image_media_when_requested(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
image_a = tmp_path / "chart_a.png"
|
||||
image_b = tmp_path / "chart b.png"
|
||||
_write_png(image_a, color=(255, 0, 0))
|
||||
_write_png(image_b, color=(0, 255, 0))
|
||||
data_url = "data:image/png;base64,Zm9v"
|
||||
remote_url = "https://example.com/chart.png"
|
||||
original_fetch_image = datasets_module.fetch_image
|
||||
|
||||
def fake_fetch_image(image_url: str) -> Image.Image:
|
||||
if image_url == remote_url:
|
||||
return Image.new("RGB", (1, 1), color=(0, 0, 255))
|
||||
return original_fetch_image(image_url)
|
||||
|
||||
monkeypatch.setattr(datasets_module, "fetch_image", fake_fetch_image)
|
||||
|
||||
jsonl = tmp_path / "images.jsonl"
|
||||
_write_jsonl(
|
||||
jsonl,
|
||||
[
|
||||
{
|
||||
"prompt": "Compare the charts.",
|
||||
"image_files": [
|
||||
str(image_a),
|
||||
image_b.as_uri(),
|
||||
remote_url,
|
||||
data_url,
|
||||
],
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
|
||||
samples = dataset.sample(
|
||||
tokenizer=_tokenizer(),
|
||||
num_requests=1,
|
||||
output_len=32,
|
||||
ensure_client_side_data=True,
|
||||
)
|
||||
|
||||
assert len(samples) == 1
|
||||
assert isinstance(samples[0].multi_modal_data, list)
|
||||
image_urls = [part["image_url"]["url"] for part in samples[0].multi_modal_data]
|
||||
|
||||
_assert_png_data_url(image_urls[0])
|
||||
_assert_png_data_url(image_urls[1])
|
||||
_assert_png_data_url(image_urls[2])
|
||||
assert image_urls[3] == data_url
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_custom_image_dataset_encodes_interleaved_image_media(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
image_a = tmp_path / "chart_a.png"
|
||||
image_b = tmp_path / "chart_b.png"
|
||||
_write_png(image_a, color=(255, 0, 0))
|
||||
_write_png(image_b, color=(0, 255, 0))
|
||||
jsonl = tmp_path / "images.jsonl"
|
||||
_write_jsonl(
|
||||
jsonl,
|
||||
[
|
||||
{
|
||||
"content": [
|
||||
{"type": "text", "text": "Compare "},
|
||||
{"type": "image", "image": str(image_a)},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": image_b.as_uri(),
|
||||
"detail": "low",
|
||||
},
|
||||
},
|
||||
],
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
|
||||
samples = dataset.sample(
|
||||
tokenizer=_tokenizer(),
|
||||
num_requests=1,
|
||||
output_len=32,
|
||||
ensure_client_side_data=True,
|
||||
)
|
||||
|
||||
sample = samples[0]
|
||||
assert isinstance(sample.prompt, list)
|
||||
parts = []
|
||||
for part in sample.prompt:
|
||||
assert isinstance(part, dict)
|
||||
parts.append(part)
|
||||
_assert_png_data_url(parts[1]["image_url"]["url"])
|
||||
_assert_png_data_url(parts[2]["image_url"]["url"])
|
||||
assert parts[2]["image_url"]["detail"] == "low"
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_custom_image_dataset_rejects_invalid_image_media(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
invalid_image = tmp_path / "not_an_image.png"
|
||||
invalid_image.write_text("not an image")
|
||||
jsonl = tmp_path / "images.jsonl"
|
||||
_write_jsonl(
|
||||
jsonl,
|
||||
[{"prompt": "Describe the image.", "image_files": [str(invalid_image)]}],
|
||||
)
|
||||
|
||||
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
|
||||
with pytest.raises(ValueError, match="Invalid image URL"):
|
||||
dataset.sample(
|
||||
tokenizer=_tokenizer(),
|
||||
num_requests=1,
|
||||
output_len=32,
|
||||
ensure_client_side_data=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_custom_image_dataset_rejects_invalid_content_part(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
jsonl = tmp_path / "images.jsonl"
|
||||
_write_jsonl(jsonl, [{"content": [{"type": "audio", "audio": "clip.wav"}]}])
|
||||
|
||||
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
|
||||
with pytest.raises(ValueError, match="type 'text', 'image', or 'image_url'"):
|
||||
dataset.sample(
|
||||
tokenizer=_tokenizer(),
|
||||
num_requests=1,
|
||||
output_len=32,
|
||||
)
|
||||
@@ -1,510 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Timing-accounting invariants for the streaming benchmark request functions.
|
||||
|
||||
serve.py derives TPOT as (latency - ttft) / (output_len - 1), which only means
|
||||
"average inter-token latency" if latency - ttft == sum(itl). These tests pin
|
||||
that identity, and pin that the trailing choice-less usage chunk does not
|
||||
extend the measured end of the request.
|
||||
|
||||
Driven against a fake session and a scripted clock: no GPU, model, or socket.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import aiohttp
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
import vllm.benchmarks.lib.endpoint_request_func as request_func_module
|
||||
from vllm.benchmarks.lib.endpoint_request_func import (
|
||||
RequestFuncInput,
|
||||
async_request_openai_audio,
|
||||
async_request_openai_chat_completions,
|
||||
async_request_openai_completions,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
|
||||
AUDIO_SAMPLE_RATE = 16_000
|
||||
|
||||
|
||||
class _ScriptedClock:
|
||||
"""A deterministic stand-in for time.perf_counter.
|
||||
|
||||
Each read advances the clock by exactly one tick, so what a code path
|
||||
observes depends only on how many times it reads the clock. A stray or
|
||||
misattributed read then shows up as a whole tick of drift rather than as
|
||||
sub-microsecond jitter, which keeps these assertions free of flakiness.
|
||||
"""
|
||||
|
||||
def __init__(self, tick: float = 1.0, start: float = 1000.0) -> None:
|
||||
self.tick = tick
|
||||
self._now = start
|
||||
|
||||
def __call__(self) -> float:
|
||||
value = self._now
|
||||
self._now += self.tick
|
||||
return value
|
||||
|
||||
|
||||
class _FakeTime:
|
||||
"""Namespace substituted for the module-level time import."""
|
||||
|
||||
def __init__(self, perf_counter) -> None:
|
||||
self.perf_counter = perf_counter
|
||||
|
||||
|
||||
def _sse(payload: dict[str, Any]) -> bytes:
|
||||
return f"data: {json.dumps(payload)}\n\n".encode()
|
||||
|
||||
|
||||
def _token_chunk(index: int, *, chat: bool) -> bytes:
|
||||
if chat:
|
||||
choice = {"index": 0, "delta": {"content": f"tok{index} "}}
|
||||
else:
|
||||
choice = {"index": 0, "text": f"tok{index} "}
|
||||
return _sse({"id": "cmpl-test", "object": "x", "choices": [choice]})
|
||||
|
||||
|
||||
def _usage_chunk(n_tokens: int) -> bytes:
|
||||
return _sse(
|
||||
{
|
||||
"id": "cmpl-test",
|
||||
"object": "x",
|
||||
"choices": [],
|
||||
"usage": {
|
||||
"prompt_tokens": 7,
|
||||
"completion_tokens": n_tokens,
|
||||
"total_tokens": 7 + n_tokens,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _build_stream(
|
||||
n_tokens: int,
|
||||
*,
|
||||
chat: bool,
|
||||
ttft_delay: float = 0.0,
|
||||
itl_delay: float = 0.0,
|
||||
usage_delay: float = 0.0,
|
||||
) -> list[tuple[float, bytes]]:
|
||||
"""Build ``(delay_before_chunk, payload)`` pairs for one streamed response.
|
||||
|
||||
``vllm bench serve`` always requests ``stream_options.include_usage``, so a
|
||||
choice-less usage chunk always trails the final token.
|
||||
"""
|
||||
chunks = [
|
||||
(ttft_delay if i == 0 else itl_delay, _token_chunk(i, chat=chat))
|
||||
for i in range(n_tokens)
|
||||
]
|
||||
chunks.append((usage_delay, _usage_chunk(n_tokens)))
|
||||
chunks.append((0.0, b"data: [DONE]\n\n"))
|
||||
return chunks
|
||||
|
||||
|
||||
class _FakeContent:
|
||||
def __init__(self, chunks: list[tuple[float, bytes]]) -> None:
|
||||
self._chunks = chunks
|
||||
|
||||
async def iter_any(self):
|
||||
for delay, payload in self._chunks:
|
||||
if delay:
|
||||
await asyncio.sleep(delay)
|
||||
yield payload
|
||||
|
||||
# The legacy script iterates ``response.content`` directly.
|
||||
def __aiter__(self):
|
||||
return self.iter_any()
|
||||
|
||||
|
||||
class _FakeResponse:
|
||||
def __init__(self, chunks: list[tuple[float, bytes]]) -> None:
|
||||
self.status = 200
|
||||
self.reason = "OK"
|
||||
self.content = _FakeContent(chunks)
|
||||
|
||||
async def __aenter__(self) -> "_FakeResponse":
|
||||
return self
|
||||
|
||||
async def __aexit__(self, exc_type, exc, tb) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
class _FakeSession:
|
||||
"""Returns a scripted SSE stream for any request, without touching a socket."""
|
||||
|
||||
def __init__(self, chunks: list[tuple[float, bytes]]) -> None:
|
||||
self._chunks = chunks
|
||||
|
||||
def post(self, **kwargs: Any) -> _FakeResponse:
|
||||
del kwargs
|
||||
return _FakeResponse(self._chunks)
|
||||
|
||||
async def __aenter__(self) -> "_FakeSession":
|
||||
return self
|
||||
|
||||
async def __aexit__(self, exc_type, exc, tb) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
# (id, request function, api url, is_chat_payload)
|
||||
STREAMING_ENDPOINTS = [
|
||||
(
|
||||
"completions",
|
||||
async_request_openai_completions,
|
||||
"http://test/v1/completions",
|
||||
False,
|
||||
),
|
||||
(
|
||||
"chat",
|
||||
async_request_openai_chat_completions,
|
||||
"http://test/v1/chat/completions",
|
||||
True,
|
||||
),
|
||||
(
|
||||
"audio",
|
||||
async_request_openai_audio,
|
||||
"http://test/v1/audio/transcriptions",
|
||||
True,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _make_input(api_url: str, n_tokens: int, endpoint_id: str) -> RequestFuncInput:
|
||||
multi_modal_content = None
|
||||
if endpoint_id == "audio":
|
||||
# A short silent clip; only its duration is read by the request func.
|
||||
multi_modal_content = {
|
||||
"audio": (
|
||||
np.zeros(AUDIO_SAMPLE_RATE // 10, dtype=np.float32),
|
||||
AUDIO_SAMPLE_RATE,
|
||||
)
|
||||
}
|
||||
return RequestFuncInput(
|
||||
prompt="hello",
|
||||
api_url=api_url,
|
||||
prompt_len=7,
|
||||
output_len=n_tokens,
|
||||
model="test-model",
|
||||
multi_modal_content=multi_modal_content,
|
||||
)
|
||||
|
||||
|
||||
def _run(
|
||||
request_func,
|
||||
api_url: str,
|
||||
endpoint_id: str,
|
||||
chunks,
|
||||
n_tokens: int,
|
||||
*,
|
||||
expect_success: bool = True,
|
||||
):
|
||||
output = asyncio.run(
|
||||
request_func(
|
||||
_make_input(api_url, n_tokens, endpoint_id),
|
||||
_FakeSession(chunks),
|
||||
)
|
||||
)
|
||||
if expect_success:
|
||||
assert output.success, output.error
|
||||
return output
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"endpoint_id,request_func,api_url,chat",
|
||||
STREAMING_ENDPOINTS,
|
||||
ids=[e[0] for e in STREAMING_ENDPOINTS],
|
||||
)
|
||||
def test_decode_span_identity_is_exact(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
endpoint_id: str,
|
||||
request_func,
|
||||
api_url: str,
|
||||
chat: bool,
|
||||
) -> None:
|
||||
"""Latency - ttft must equal sum(itl) exactly.
|
||||
|
||||
On a scripted clock this can only drift if a code path reads the clock an
|
||||
extra time or attributes a read to the wrong event.
|
||||
"""
|
||||
tick = 1.0
|
||||
monkeypatch.setattr(
|
||||
request_func_module, "time", _FakeTime(_ScriptedClock(tick=tick))
|
||||
)
|
||||
|
||||
n_tokens = 5
|
||||
chunks = _build_stream(n_tokens, chat=chat)
|
||||
output = _run(request_func, api_url, endpoint_id, chunks, n_tokens)
|
||||
|
||||
assert len(output.itl) == n_tokens - 1
|
||||
assert output.output_tokens == n_tokens
|
||||
|
||||
residual = (output.latency - output.ttft) - sum(output.itl)
|
||||
assert residual == pytest.approx(0.0, abs=tick / 1000), (
|
||||
f"{endpoint_id}: (latency - ttft) - sum(itl) = {residual!r}; "
|
||||
f"TPOT is derived from latency - ttft, so it must match the measured "
|
||||
f"inter-token latencies"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"endpoint_id,request_func,api_url,chat",
|
||||
STREAMING_ENDPOINTS,
|
||||
ids=[e[0] for e in STREAMING_ENDPOINTS],
|
||||
)
|
||||
def test_trailing_usage_chunk_does_not_extend_latency(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
endpoint_id: str,
|
||||
request_func,
|
||||
api_url: str,
|
||||
chat: bool,
|
||||
) -> None:
|
||||
"""E2E latency must end at the last token, not at the usage trailer.
|
||||
|
||||
The usage chunk carries no token, so counting it stretches E2EL (and
|
||||
therefore TPOT) by the delivery time of one extra SSE frame. The [DONE]
|
||||
sentinel is already excluded on the same grounds.
|
||||
"""
|
||||
tick = 1.0
|
||||
clock = _ScriptedClock(tick=tick)
|
||||
monkeypatch.setattr(request_func_module, "time", _FakeTime(clock))
|
||||
|
||||
n_tokens = 5
|
||||
chunks = _build_stream(n_tokens, chat=chat)
|
||||
output = _run(request_func, api_url, endpoint_id, chunks, n_tokens)
|
||||
|
||||
# Clock reads: 1 for the start time, then 1 per non-[DONE] chunk. The last
|
||||
# token is therefore read at start + n_tokens ticks; the usage chunk one
|
||||
# tick later.
|
||||
expected_latency = n_tokens * tick
|
||||
assert output.latency == pytest.approx(expected_latency, abs=tick / 1000), (
|
||||
f"{endpoint_id}: latency {output.latency} should stop at the final "
|
||||
f"token ({expected_latency}), not run on to the usage chunk "
|
||||
f"({expected_latency + tick})"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"endpoint_id,request_func,api_url,chat",
|
||||
STREAMING_ENDPOINTS,
|
||||
ids=[e[0] for e in STREAMING_ENDPOINTS],
|
||||
)
|
||||
def test_decode_span_identity_holds_with_real_clock(
|
||||
endpoint_id: str,
|
||||
request_func,
|
||||
api_url: str,
|
||||
chat: bool,
|
||||
) -> None:
|
||||
"""The identity also holds against the real clock, with real gaps.
|
||||
|
||||
The residual differences out the same recorded timestamps, so wall clock
|
||||
jitter cancels and only float rounding remains; a microsecond bound is
|
||||
ample rather than tight.
|
||||
"""
|
||||
n_tokens = 4
|
||||
chunks = _build_stream(
|
||||
n_tokens, chat=chat, ttft_delay=0.02, itl_delay=0.01, usage_delay=0.02
|
||||
)
|
||||
output = _run(request_func, api_url, endpoint_id, chunks, n_tokens)
|
||||
|
||||
residual = (output.latency - output.ttft) - sum(output.itl)
|
||||
assert abs(residual) < 1e-6, (
|
||||
f"{endpoint_id}: (latency - ttft) - sum(itl) drifted by {residual:.9f}s"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"endpoint_id,request_func,api_url,chat",
|
||||
STREAMING_ENDPOINTS,
|
||||
ids=[e[0] for e in STREAMING_ENDPOINTS],
|
||||
)
|
||||
def test_usage_only_stream_is_not_reported_as_success(
|
||||
endpoint_id: str,
|
||||
request_func,
|
||||
api_url: str,
|
||||
chat: bool,
|
||||
) -> None:
|
||||
"""A stream with no token chunk must fail rather than report 0 ms.
|
||||
|
||||
Only token chunks move the end of the request, so a degenerate stream
|
||||
carrying nothing but a usage trailer leaves latency at zero. Reporting that
|
||||
as a success would feed a zero-duration request into the benchmark
|
||||
aggregates.
|
||||
"""
|
||||
chunks = [(0.0, _usage_chunk(0)), (0.0, b"data: [DONE]\n\n")]
|
||||
output = _run(request_func, api_url, endpoint_id, chunks, 0, expect_success=False)
|
||||
|
||||
assert not output.success
|
||||
assert output.itl == []
|
||||
assert "TTFT" in output.error
|
||||
|
||||
|
||||
# Significant whitespace at every edge, plus a multibyte character so that a
|
||||
# split can land inside a UTF-8 sequence.
|
||||
_BOUNDARY_TEXT = " leading 中 trailing "
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"endpoint_id,request_func,api_url,chat",
|
||||
STREAMING_ENDPOINTS,
|
||||
ids=[e[0] for e in STREAMING_ENDPOINTS],
|
||||
)
|
||||
def test_streamed_output_is_invariant_to_transport_chunk_boundaries(
|
||||
endpoint_id: str,
|
||||
request_func,
|
||||
api_url: str,
|
||||
chat: bool,
|
||||
) -> None:
|
||||
"""iter_any() yields arbitrary transport fragments, so parsing must
|
||||
give the same result no matter where the byte stream is split."""
|
||||
if chat:
|
||||
choice: dict[str, Any] = {"index": 0, "delta": {"content": _BOUNDARY_TEXT}}
|
||||
else:
|
||||
choice = {"index": 0, "text": _BOUNDARY_TEXT}
|
||||
# Not _sse(): its ensure_ascii default would escape the multibyte char.
|
||||
event = json.dumps(
|
||||
{"id": "cmpl-test", "object": "x", "choices": [choice]}, ensure_ascii=False
|
||||
)
|
||||
stream = f"data: {event}\n\ndata: [DONE]\n\n".encode()
|
||||
|
||||
async def run_all_splits() -> list[tuple[int, bool, str, str]]:
|
||||
failures = []
|
||||
for split_at in range(1, len(stream)):
|
||||
chunks = [(0.0, stream[:split_at]), (0.0, stream[split_at:])]
|
||||
output = await request_func(
|
||||
_make_input(api_url, 1, endpoint_id),
|
||||
_FakeSession(chunks),
|
||||
)
|
||||
if not output.success or output.generated_text != _BOUNDARY_TEXT:
|
||||
error = output.error.splitlines()[-1] if output.error else ""
|
||||
failures.append(
|
||||
(split_at, output.success, output.generated_text, error)
|
||||
)
|
||||
return failures
|
||||
|
||||
assert asyncio.run(run_all_splits()) == []
|
||||
|
||||
|
||||
# ``benchmarks/backend_request_func.py`` is a standalone script rather than a
|
||||
# package module, so it is loaded by path. It also builds its own ClientSession,
|
||||
# so the scripted stream is injected by swapping the module's aiohttp reference.
|
||||
_LEGACY_PATH = Path(__file__).parents[2] / "benchmarks" / "backend_request_func.py"
|
||||
|
||||
|
||||
class _FakeAiohttp:
|
||||
"""Stands in for the legacy module's aiohttp import."""
|
||||
|
||||
def __init__(self, chunks: list[tuple[float, bytes]]) -> None:
|
||||
self._chunks = chunks
|
||||
|
||||
def __getattr__(self, name: str) -> Any:
|
||||
return getattr(aiohttp, name)
|
||||
|
||||
def ClientSession(self, **kwargs: Any) -> _FakeSession: # noqa: N802
|
||||
del kwargs
|
||||
return _FakeSession(self._chunks)
|
||||
|
||||
|
||||
def _load_legacy():
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"legacy_backend_request_func", _LEGACY_PATH
|
||||
)
|
||||
assert spec is not None and spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
LEGACY_ENDPOINTS = [
|
||||
(
|
||||
"completions",
|
||||
"async_request_openai_completions",
|
||||
"http://test/v1/completions",
|
||||
False,
|
||||
),
|
||||
(
|
||||
"chat",
|
||||
"async_request_openai_chat_completions",
|
||||
"http://test/v1/chat/completions",
|
||||
True,
|
||||
),
|
||||
(
|
||||
"audio",
|
||||
"async_request_openai_audio",
|
||||
"http://test/v1/audio/transcriptions",
|
||||
True,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _run_legacy(
|
||||
monkeypatch, func_name, api_url, endpoint_id, chunks, n_tokens, clock=None
|
||||
):
|
||||
module = _load_legacy()
|
||||
monkeypatch.setattr(module, "aiohttp", _FakeAiohttp(chunks))
|
||||
if clock is not None:
|
||||
monkeypatch.setattr(module, "time", _FakeTime(clock))
|
||||
request_func = getattr(module, func_name)
|
||||
return asyncio.run(request_func(_make_input(api_url, n_tokens, endpoint_id)))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"endpoint_id,func_name,api_url,chat",
|
||||
LEGACY_ENDPOINTS,
|
||||
ids=[e[0] for e in LEGACY_ENDPOINTS],
|
||||
)
|
||||
def test_legacy_decode_span_identity_is_exact(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
endpoint_id: str,
|
||||
func_name: str,
|
||||
api_url: str,
|
||||
chat: bool,
|
||||
) -> None:
|
||||
"""benchmarks/backend_request_func.py must honour the same identity."""
|
||||
tick = 1.0
|
||||
n_tokens = 5
|
||||
chunks = _build_stream(n_tokens, chat=chat)
|
||||
output = _run_legacy(
|
||||
monkeypatch,
|
||||
func_name,
|
||||
api_url,
|
||||
endpoint_id,
|
||||
chunks,
|
||||
n_tokens,
|
||||
clock=_ScriptedClock(tick=tick),
|
||||
)
|
||||
|
||||
assert output.success, output.error
|
||||
assert len(output.itl) == n_tokens - 1
|
||||
|
||||
residual = (output.latency - output.ttft) - sum(output.itl)
|
||||
assert residual == pytest.approx(0.0, abs=tick / 1000), (
|
||||
f"legacy {endpoint_id}: (latency - ttft) - sum(itl) = {residual!r}"
|
||||
)
|
||||
assert output.latency == pytest.approx(n_tokens * tick, abs=tick / 1000)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"endpoint_id,func_name,api_url,chat",
|
||||
LEGACY_ENDPOINTS,
|
||||
ids=[e[0] for e in LEGACY_ENDPOINTS],
|
||||
)
|
||||
def test_legacy_usage_only_stream_is_not_reported_as_success(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
endpoint_id: str,
|
||||
func_name: str,
|
||||
api_url: str,
|
||||
chat: bool,
|
||||
) -> None:
|
||||
"""A usage-only stream must fail rather than report a zero-duration success."""
|
||||
chunks = [(0.0, _usage_chunk(0)), (0.0, b"data: [DONE]\n\n")]
|
||||
output = _run_legacy(monkeypatch, func_name, api_url, endpoint_id, chunks, 0)
|
||||
|
||||
assert not output.success
|
||||
assert output.itl == []
|
||||
@@ -1,178 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import json
|
||||
import subprocess
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm.benchmarks.forward.batch import build_batch
|
||||
from vllm.benchmarks.forward.main import run_case
|
||||
from vllm.benchmarks.forward.profile import summarize
|
||||
from vllm.benchmarks.forward.shapes import jitter_kv_lens, parse_case
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
MODEL_NAME = "meta-llama/Llama-3.2-1B-Instruct"
|
||||
|
||||
|
||||
def _group(block_size: int, replay: int = 0, dcp_sharded: bool = False):
|
||||
return SimpleNamespace(
|
||||
kv_cache_spec=SimpleNamespace(
|
||||
block_size=block_size,
|
||||
prefix_replay_tokens=replay,
|
||||
dcp_sharded=dcp_sharded,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_build_batch_layout():
|
||||
"""The runner sees each request kind as the scheduler would schedule it."""
|
||||
case = parse_case("c=2*1@101,4@64:3,32,8@48c,16@316")
|
||||
groups = [_group(16), _group(32, replay=128)]
|
||||
batch = build_batch(case, groups, 1000, 1000, np.random.default_rng(0))
|
||||
|
||||
# (scheduled, computed, is_decode) per request: decode, verify, prefill,
|
||||
# chunked prefill, then a prefix hit replaying the last 128 hit tokens.
|
||||
assert list(batch.expected.values()) == [
|
||||
(1, 100, True),
|
||||
(1, 100, True),
|
||||
(4, 60, True),
|
||||
(32, 0, False),
|
||||
(8, 40, False),
|
||||
(16 + 128, 300 - 128, False),
|
||||
]
|
||||
output = batch.scheduler_output
|
||||
assert output.total_num_scheduled_tokens == 1 + 1 + 4 + 32 + 8 + 144
|
||||
assert len(output.scheduled_spec_decode_tokens["bench-2"]) == 3
|
||||
assert output.scheduled_new_reqs[-1].replay_start == 300 - 128
|
||||
for req_id, (scheduled, _, _) in batch.expected.items():
|
||||
assert len(batch.query_token_ids[req_id]) == scheduled
|
||||
# Groups share one block pool: block IDs must not overlap.
|
||||
ranges = sorted(r for group in batch.group_block_ranges for r in group)
|
||||
assert ranges[0][0] == 1
|
||||
assert all(a[1] == b[0] for a, b in zip(ranges, ranges[1:]))
|
||||
|
||||
with pytest.raises(ValueError, match="KV blocks"):
|
||||
build_batch(case, groups, 10, 1000, np.random.default_rng(0))
|
||||
|
||||
# DCP spreads each block's tokens over the DCP ranks, like the scheduler.
|
||||
groups = [_group(16, dcp_sharded=True), _group(16)]
|
||||
batch = build_batch(
|
||||
parse_case("c=1@640"), groups, 1000, 1000, np.random.default_rng(0), 4
|
||||
)
|
||||
assert batch.group_block_ranges == [[(1, 11)], [(11, 51)]]
|
||||
|
||||
|
||||
def test_jitter_kv_lens():
|
||||
"""Prefixes vary, capped at the max length; queries and fresh prefills don't.
|
||||
A case's draw depends only on the seed and its name, so adding or reordering
|
||||
other cases doesn't change it."""
|
||||
case = parse_case("c=1000*1@1001,4@64:3,32")
|
||||
jittered = jitter_kv_lens(case, 0.5, 1400, 0)
|
||||
assert jitter_kv_lens(case, 0.5, 1400, 0) == jittered
|
||||
other = jitter_kv_lens(replace(case, name="d"), 0.5, 1400, 0)
|
||||
assert other.requests != jittered.requests
|
||||
prefixes = np.array([r.prefix_len for r in jittered.requests[:1000]])
|
||||
assert prefixes.min() >= 500 and max(r.kv_len for r in jittered.requests) == 1400
|
||||
assert prefixes.std() > 200
|
||||
assert [r.query_len for r in jittered.requests] == [
|
||||
r.query_len for r in case.requests
|
||||
]
|
||||
assert jittered.requests[-1] == case.requests[-1]
|
||||
assert jitter_kv_lens(case, 0, 2048, 0) == case
|
||||
with pytest.raises(ValueError, match="KV length"):
|
||||
jitter_kv_lens(case, 0, 1000, 0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("capacity", [0, 2])
|
||||
def test_bench_forward_rejects_oversized_verify(capacity):
|
||||
"""Reject drafts beyond the runner's buffer before launching CUDA kernels."""
|
||||
runner = SimpleNamespace(num_speculative_steps=capacity)
|
||||
case = parse_case("verify=2*4@64:3")
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match=rf"Case 'verify' needs 3 draft tokens.*supports {capacity}",
|
||||
):
|
||||
run_case(runner, case, SimpleNamespace())
|
||||
|
||||
|
||||
def test_profile_summary():
|
||||
"""Each replay's leading restore/rendezvous activities are left out, and
|
||||
kernels are grouped by name without template or function arguments."""
|
||||
|
||||
def act(name, ts, dur, cat="kernel"):
|
||||
return {"cat": cat, "name": name, "ts": ts, "dur": dur}
|
||||
|
||||
replay = [
|
||||
("Memcpy DtoD (Device -> Device)", "gpu_memcpy", 5),
|
||||
("void ns::gemm<128>(Params)", "kernel", 10),
|
||||
("void ns::gemm<64>(Params)", "kernel", 20),
|
||||
("std::enable_if<(1>0), void>::type ns::norm<float>(float*)", "kernel", 4),
|
||||
]
|
||||
events = [{"cat": "cpu_op", "name": "x", "ts": 0, "dur": 1}]
|
||||
for r in range(2):
|
||||
for i, (name, cat, dur) in enumerate(replay):
|
||||
events.append(act(name, 1000 * r + 100 * i, dur, cat))
|
||||
summary = summarize(events, replays=2, skip=1)
|
||||
assert summary["pre_forward_us"] == 5
|
||||
assert summary["busy_us"] == 34
|
||||
assert summary["span_us"] == 204
|
||||
assert [(k["name"], k["us"], k["calls"]) for k in summary["kernels"]] == [
|
||||
("ns::gemm", 30, 2),
|
||||
("ns::norm", 4, 1),
|
||||
]
|
||||
with pytest.raises(ValueError, match="replays"):
|
||||
summarize(events[:-1], replays=2, skip=1)
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
# TODO(woosuk): Support ROCm and re-enable this test there.
|
||||
@pytest.mark.skipif(
|
||||
current_platform.is_rocm(), reason="Hangs on ROCm; the bench is CUDA-only."
|
||||
)
|
||||
def test_bench_forward(tmp_path):
|
||||
output = tmp_path / "forward.json"
|
||||
command = [
|
||||
"vllm",
|
||||
"bench",
|
||||
"forward",
|
||||
"--model",
|
||||
MODEL_NAME,
|
||||
"--max-model-len",
|
||||
"2048",
|
||||
"--gpu-memory-utilization",
|
||||
"0.5",
|
||||
"--case",
|
||||
"decode=8*1@512",
|
||||
"--case",
|
||||
"prefill=256",
|
||||
"--case",
|
||||
"mix=128@256,4*1@512",
|
||||
"--iters",
|
||||
"3",
|
||||
"--samples",
|
||||
"2",
|
||||
"--output-json",
|
||||
str(output),
|
||||
"--profile-dir",
|
||||
str(tmp_path / "profile"),
|
||||
]
|
||||
result = subprocess.run(command, capture_output=True, text=True)
|
||||
print(result.stdout)
|
||||
print(result.stderr)
|
||||
assert result.returncode == 0, f"Benchmark failed: {result.stderr}"
|
||||
|
||||
results = {row["case"]: row for row in json.loads(output.read_text())}
|
||||
assert list(results) == ["decode", "prefill", "mix"]
|
||||
for row in results.values():
|
||||
rank0 = row["ranks"][0]
|
||||
assert len(rank0["batch_order"]) == len(row["requests"])
|
||||
assert rank0["finite"] and rank0["graph_deterministic"]
|
||||
assert row["median_us"] > 0
|
||||
profile = rank0["profile"]
|
||||
assert profile["top_kernels"] and profile["span_us"] > 0
|
||||
summary = json.loads(Path(profile["summary"]).read_text())
|
||||
assert Path(summary["trace"]).exists()
|
||||
@@ -1,171 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pandas as pd
|
||||
import pytest
|
||||
|
||||
from vllm.benchmarks.sweep.plot import (
|
||||
PlotEqualTo,
|
||||
PlotFilterBase,
|
||||
PlotFilters,
|
||||
PlotGreaterThan,
|
||||
PlotGreaterThanOrEqualTo,
|
||||
PlotLessThan,
|
||||
PlotLessThanOrEqualTo,
|
||||
PlotNotEqualTo,
|
||||
)
|
||||
|
||||
|
||||
class TestPlotFilters:
|
||||
"""Test PlotFilter functionality including 'inf' edge case."""
|
||||
|
||||
def setup_method(self):
|
||||
"""Create sample DataFrames for testing."""
|
||||
# DataFrame with numeric values
|
||||
self.df_numeric = pd.DataFrame(
|
||||
{
|
||||
"request_rate": [1.0, 5.0, 10.0, 50.0, 100.0],
|
||||
"value": [10, 20, 30, 40, 50],
|
||||
}
|
||||
)
|
||||
|
||||
# DataFrame with float('inf') - note: string "inf" values are coerced
|
||||
# to float when loading data, so we only test with float('inf')
|
||||
self.df_inf_float = pd.DataFrame(
|
||||
{
|
||||
"request_rate": [1.0, 5.0, 10.0, float("inf"), float("inf")],
|
||||
"value": [10, 20, 30, 40, 50],
|
||||
}
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"target,expected_count",
|
||||
[
|
||||
("5.0", 1),
|
||||
("10.0", 1),
|
||||
("1.0", 1),
|
||||
],
|
||||
)
|
||||
def test_equal_to_numeric(self, target, expected_count):
|
||||
"""Test PlotEqualTo with numeric values."""
|
||||
filter_obj = PlotEqualTo("request_rate", target)
|
||||
result = filter_obj.apply(self.df_numeric)
|
||||
assert len(result) == expected_count
|
||||
|
||||
def test_equal_to_inf_float(self):
|
||||
"""Test PlotEqualTo with float('inf')."""
|
||||
filter_obj = PlotEqualTo("request_rate", "inf")
|
||||
result = filter_obj.apply(self.df_inf_float)
|
||||
# Should match both float('inf') entries because float('inf') == float('inf')
|
||||
assert len(result) == 2
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"target,expected_count",
|
||||
[
|
||||
("5.0", 4), # All except 5.0
|
||||
("1.0", 4), # All except 1.0
|
||||
],
|
||||
)
|
||||
def test_not_equal_to_numeric(self, target, expected_count):
|
||||
"""Test PlotNotEqualTo with numeric values."""
|
||||
filter_obj = PlotNotEqualTo("request_rate", target)
|
||||
result = filter_obj.apply(self.df_numeric)
|
||||
assert len(result) == expected_count
|
||||
|
||||
def test_not_equal_to_inf_float(self):
|
||||
"""Test PlotNotEqualTo with float('inf')."""
|
||||
filter_obj = PlotNotEqualTo("request_rate", "inf")
|
||||
result = filter_obj.apply(self.df_inf_float)
|
||||
# Should exclude float('inf') entries
|
||||
assert len(result) == 3
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"target,expected_count",
|
||||
[
|
||||
("10.0", 2), # 1.0, 5.0
|
||||
("50.0", 3), # 1.0, 5.0, 10.0
|
||||
("5.0", 1), # 1.0
|
||||
],
|
||||
)
|
||||
def test_less_than(self, target, expected_count):
|
||||
"""Test PlotLessThan with numeric values."""
|
||||
filter_obj = PlotLessThan("request_rate", target)
|
||||
result = filter_obj.apply(self.df_numeric)
|
||||
assert len(result) == expected_count
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"target,expected_count",
|
||||
[
|
||||
("10.0", 3), # 1.0, 5.0, 10.0
|
||||
("5.0", 2), # 1.0, 5.0
|
||||
],
|
||||
)
|
||||
def test_less_than_or_equal_to(self, target, expected_count):
|
||||
"""Test PlotLessThanOrEqualTo with numeric values."""
|
||||
filter_obj = PlotLessThanOrEqualTo("request_rate", target)
|
||||
result = filter_obj.apply(self.df_numeric)
|
||||
assert len(result) == expected_count
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"target,expected_count",
|
||||
[
|
||||
("10.0", 2), # 50.0, 100.0
|
||||
("5.0", 3), # 10.0, 50.0, 100.0
|
||||
],
|
||||
)
|
||||
def test_greater_than(self, target, expected_count):
|
||||
"""Test PlotGreaterThan with numeric values."""
|
||||
filter_obj = PlotGreaterThan("request_rate", target)
|
||||
result = filter_obj.apply(self.df_numeric)
|
||||
assert len(result) == expected_count
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"target,expected_count",
|
||||
[
|
||||
("10.0", 3), # 10.0, 50.0, 100.0
|
||||
("5.0", 4), # 5.0, 10.0, 50.0, 100.0
|
||||
],
|
||||
)
|
||||
def test_greater_than_or_equal_to(self, target, expected_count):
|
||||
"""Test PlotGreaterThanOrEqualTo with numeric values."""
|
||||
filter_obj = PlotGreaterThanOrEqualTo("request_rate", target)
|
||||
result = filter_obj.apply(self.df_numeric)
|
||||
assert len(result) == expected_count
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"filter_str,expected_var,expected_target,expected_type",
|
||||
[
|
||||
("request_rate==5.0", "request_rate", "5.0", PlotEqualTo),
|
||||
("request_rate!=10.0", "request_rate", "10.0", PlotNotEqualTo),
|
||||
("request_rate<50.0", "request_rate", "50.0", PlotLessThan),
|
||||
("request_rate<=50.0", "request_rate", "50.0", PlotLessThanOrEqualTo),
|
||||
("request_rate>10.0", "request_rate", "10.0", PlotGreaterThan),
|
||||
("request_rate>=10.0", "request_rate", "10.0", PlotGreaterThanOrEqualTo),
|
||||
("request_rate==inf", "request_rate", "inf", PlotEqualTo),
|
||||
("request_rate!='inf'", "request_rate", "inf", PlotNotEqualTo),
|
||||
],
|
||||
)
|
||||
def test_parse_str(self, filter_str, expected_var, expected_target, expected_type):
|
||||
"""Test parsing filter strings."""
|
||||
filter_obj = PlotFilterBase.parse_str(filter_str)
|
||||
assert isinstance(filter_obj, expected_type)
|
||||
assert filter_obj.var == expected_var
|
||||
assert filter_obj.target == expected_target
|
||||
|
||||
def test_parse_str_inf_edge_case(self):
|
||||
"""Test parsing 'inf' string in filter."""
|
||||
filter_obj = PlotFilterBase.parse_str("request_rate==inf")
|
||||
assert isinstance(filter_obj, PlotEqualTo)
|
||||
assert filter_obj.var == "request_rate"
|
||||
assert filter_obj.target == "inf"
|
||||
|
||||
def test_parse_multiple_filters(self):
|
||||
"""Test parsing multiple filters."""
|
||||
filters = PlotFilters.parse_str("request_rate>5.0,value<=40")
|
||||
assert len(filters) == 2
|
||||
assert isinstance(filters[0], PlotGreaterThan)
|
||||
assert isinstance(filters[1], PlotLessThanOrEqualTo)
|
||||
|
||||
def test_parse_empty_filter(self):
|
||||
"""Test parsing empty filter string."""
|
||||
filters = PlotFilters.parse_str("")
|
||||
assert len(filters) == 0
|
||||
@@ -1,486 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import random
|
||||
from typing import Any, NamedTuple, cast
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
from transformers import AutoTokenizer, PreTrainedTokenizerBase
|
||||
|
||||
from vllm.benchmarks.datasets import (
|
||||
RandomDataset,
|
||||
RandomMultiModalDataset,
|
||||
SampleRequest,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def hf_tokenizer() -> PreTrainedTokenizerBase:
|
||||
# Use a small, commonly available tokenizer
|
||||
return AutoTokenizer.from_pretrained("openai-community/gpt2")
|
||||
|
||||
|
||||
class Params(NamedTuple):
|
||||
num_requests: int
|
||||
prefix_len: int
|
||||
range_ratio: float
|
||||
input_len: int
|
||||
output_len: int
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def random_dataset_params() -> Params:
|
||||
return Params(
|
||||
num_requests=16, prefix_len=7, range_ratio=0.3, input_len=50, output_len=20
|
||||
)
|
||||
|
||||
|
||||
def _fingerprint_sample(req: SampleRequest) -> tuple[str, int, int]:
|
||||
"""Project a SampleRequest into a comparable tuple."""
|
||||
assert isinstance(req.prompt, str)
|
||||
return (req.prompt, req.prompt_len, req.expected_output_len)
|
||||
|
||||
|
||||
def _collect_samples(
|
||||
dataset: RandomDataset,
|
||||
tokenizer: PreTrainedTokenizerBase,
|
||||
num_requests: int = 16,
|
||||
prefix_len: int = 7,
|
||||
range_ratio: float = 0.3,
|
||||
input_len: int = 50,
|
||||
output_len: int = 20,
|
||||
) -> list[tuple[str, int, int]]:
|
||||
samples = dataset.sample(
|
||||
tokenizer=tokenizer,
|
||||
num_requests=num_requests,
|
||||
prefix_len=prefix_len,
|
||||
range_ratio=range_ratio,
|
||||
input_len=input_len,
|
||||
output_len=output_len,
|
||||
)
|
||||
return [_fingerprint_sample(s) for s in samples]
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_random_dataset_same_seed(
|
||||
hf_tokenizer: PreTrainedTokenizerBase, random_dataset_params: Params
|
||||
) -> None:
|
||||
"""Same seed should yield identical outputs, even if global RNGs change.
|
||||
|
||||
This guards against accidental reliance on Python's random or np.random
|
||||
in RandomDataset after moving to numpy.default_rng.
|
||||
"""
|
||||
p = random_dataset_params
|
||||
common_seed = 123
|
||||
dataset_a = RandomDataset(random_seed=common_seed)
|
||||
dataset_b = RandomDataset(random_seed=common_seed)
|
||||
a = _collect_samples(
|
||||
dataset_a,
|
||||
hf_tokenizer,
|
||||
num_requests=p.num_requests,
|
||||
prefix_len=p.prefix_len,
|
||||
range_ratio=p.range_ratio,
|
||||
input_len=p.input_len,
|
||||
output_len=p.output_len,
|
||||
)
|
||||
|
||||
# Perturb global RNG state to ensure isolation
|
||||
random.seed(999)
|
||||
_ = [random.random() for _ in range(100)]
|
||||
np.random.seed(888)
|
||||
_ = [np.random.random() for _ in range(100)]
|
||||
|
||||
b = _collect_samples(
|
||||
dataset_b,
|
||||
hf_tokenizer,
|
||||
num_requests=p.num_requests,
|
||||
prefix_len=p.prefix_len,
|
||||
range_ratio=p.range_ratio,
|
||||
input_len=p.input_len,
|
||||
output_len=p.output_len,
|
||||
)
|
||||
assert a == b
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_random_dataset_different_seeds(
|
||||
hf_tokenizer: PreTrainedTokenizerBase, random_dataset_params: Params
|
||||
) -> None:
|
||||
"""Different seeds should change outputs with overwhelming likelihood."""
|
||||
p = random_dataset_params
|
||||
seed_a = 0
|
||||
dataset_a = RandomDataset(random_seed=seed_a)
|
||||
a = _collect_samples(
|
||||
dataset_a,
|
||||
hf_tokenizer,
|
||||
num_requests=p.num_requests,
|
||||
prefix_len=p.prefix_len,
|
||||
range_ratio=p.range_ratio,
|
||||
input_len=p.input_len,
|
||||
output_len=p.output_len,
|
||||
)
|
||||
|
||||
seed_b = 999
|
||||
dataset_b = RandomDataset(random_seed=seed_b)
|
||||
# Perturb global RNG with same seed as dataset_a to ensure isolation
|
||||
random.seed(seed_a)
|
||||
np.random.seed(seed_a)
|
||||
b = _collect_samples(
|
||||
dataset_b,
|
||||
hf_tokenizer,
|
||||
num_requests=p.num_requests,
|
||||
prefix_len=p.prefix_len,
|
||||
range_ratio=p.range_ratio,
|
||||
input_len=p.input_len,
|
||||
output_len=p.output_len,
|
||||
)
|
||||
assert a != b
|
||||
|
||||
|
||||
# -----------------------------
|
||||
# RandomMultiModalDataset tests
|
||||
# -----------------------------
|
||||
|
||||
|
||||
def _mm_fingerprint_sample(
|
||||
req: SampleRequest,
|
||||
) -> tuple[str, int, int, int, list[str]]:
|
||||
"""Create a compact fingerprint for multimodal samples.
|
||||
|
||||
Includes:
|
||||
- prompt string
|
||||
- prompt_len
|
||||
- expected_output_len
|
||||
- count of multimodal items
|
||||
- per-item type and URL prefix (e.g., 'data:image/jpeg;base64,')
|
||||
"""
|
||||
assert isinstance(req.prompt, str)
|
||||
items = req.multi_modal_data or []
|
||||
item_prefixes: list[str] = []
|
||||
for it in items:
|
||||
if isinstance(it, dict) and it.get("type") == "image_url":
|
||||
url = it.get("image_url", {}).get("url", "")
|
||||
# Only keep a short identifying prefix to avoid huge strings
|
||||
item_prefixes.append(f"image:{url[:22]}")
|
||||
elif isinstance(it, dict) and it.get("type") == "video_url":
|
||||
url = it.get("video_url", {}).get("url", "")
|
||||
item_prefixes.append(f"video:{url[:22]}")
|
||||
else:
|
||||
item_prefixes.append("unknown:")
|
||||
return (
|
||||
req.prompt,
|
||||
req.prompt_len,
|
||||
req.expected_output_len,
|
||||
len(items),
|
||||
item_prefixes,
|
||||
)
|
||||
|
||||
|
||||
def _collect_mm_samples(
|
||||
dataset: RandomMultiModalDataset,
|
||||
tokenizer: PreTrainedTokenizerBase,
|
||||
*,
|
||||
num_requests: int = 8,
|
||||
prefix_len: int = 3,
|
||||
range_ratio: float = 0.0,
|
||||
input_len: int = 20,
|
||||
output_len: int = 5,
|
||||
base_items_per_request: int = 2,
|
||||
num_mm_items_range_ratio: float = 0.0,
|
||||
limit_mm_per_prompt: dict[str, int] | None = None,
|
||||
bucket_config: dict[tuple[int, int, int], float] | None = None,
|
||||
enable_multimodal_chat: bool = False,
|
||||
) -> list[SampleRequest]:
|
||||
if limit_mm_per_prompt is None:
|
||||
limit_mm_per_prompt = {"image": 5, "video": 0}
|
||||
if bucket_config is None:
|
||||
bucket_config = {(32, 32, 1): 0.5, (52, 64, 1): 0.5}
|
||||
return dataset.sample(
|
||||
tokenizer=tokenizer,
|
||||
num_requests=num_requests,
|
||||
prefix_len=prefix_len,
|
||||
range_ratio=range_ratio,
|
||||
input_len=input_len,
|
||||
output_len=output_len,
|
||||
base_items_per_request=base_items_per_request,
|
||||
num_mm_items_range_ratio=num_mm_items_range_ratio,
|
||||
limit_mm_per_prompt=limit_mm_per_prompt,
|
||||
bucket_config=bucket_config,
|
||||
enable_multimodal_chat=enable_multimodal_chat,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_random_mm_same_seed(hf_tokenizer: PreTrainedTokenizerBase) -> None:
|
||||
seed = 42
|
||||
ds_a = RandomMultiModalDataset(random_seed=seed)
|
||||
ds_b = RandomMultiModalDataset(random_seed=seed)
|
||||
a = _collect_mm_samples(ds_a, hf_tokenizer)
|
||||
b = _collect_mm_samples(ds_b, hf_tokenizer)
|
||||
fa = [_mm_fingerprint_sample(s) for s in a]
|
||||
fb = [_mm_fingerprint_sample(s) for s in b]
|
||||
assert fa == fb
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_random_mm_different_seeds(
|
||||
hf_tokenizer: PreTrainedTokenizerBase,
|
||||
) -> None:
|
||||
ds_a = RandomMultiModalDataset(random_seed=0)
|
||||
ds_b = RandomMultiModalDataset(random_seed=999)
|
||||
a = _collect_mm_samples(ds_a, hf_tokenizer)
|
||||
b = _collect_mm_samples(ds_b, hf_tokenizer)
|
||||
fa = [_mm_fingerprint_sample(s) for s in a]
|
||||
fb = [_mm_fingerprint_sample(s) for s in b]
|
||||
assert fa != fb
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_random_mm_respects_limits(
|
||||
hf_tokenizer: PreTrainedTokenizerBase,
|
||||
) -> None:
|
||||
ds = RandomMultiModalDataset(random_seed=0)
|
||||
# Requesting 3 items with a per-prompt limit of 1 should error per current
|
||||
# design (dataset refuses to silently clamp below the requested baseline).
|
||||
with pytest.raises(ValueError):
|
||||
_collect_mm_samples(
|
||||
ds,
|
||||
hf_tokenizer,
|
||||
num_requests=12,
|
||||
base_items_per_request=3,
|
||||
num_mm_items_range_ratio=0.0,
|
||||
limit_mm_per_prompt={"image": 1, "video": 0},
|
||||
bucket_config={(32, 32, 1): 1.0},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_random_mm_zero_prob_entries_are_removed(
|
||||
hf_tokenizer: PreTrainedTokenizerBase,
|
||||
) -> None:
|
||||
ds = RandomMultiModalDataset(random_seed=0)
|
||||
# Second bucket has zero probability and should be ignored after
|
||||
# normalization
|
||||
samples = _collect_mm_samples(
|
||||
ds,
|
||||
hf_tokenizer,
|
||||
num_requests=6,
|
||||
base_items_per_request=2,
|
||||
num_mm_items_range_ratio=0.0,
|
||||
limit_mm_per_prompt={"image": 10, "video": 0},
|
||||
bucket_config={(32, 32, 1): 1.0, (52, 64, 1): 0.0},
|
||||
)
|
||||
for s in samples:
|
||||
assert isinstance(s.multi_modal_data, list)
|
||||
typed_mm = cast(list[dict[str, Any]], s.multi_modal_data)
|
||||
for it in typed_mm:
|
||||
assert it.get("type") == "image_url"
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_random_mm_zero_items(hf_tokenizer: PreTrainedTokenizerBase) -> None:
|
||||
ds = RandomMultiModalDataset(random_seed=0)
|
||||
samples = _collect_mm_samples(
|
||||
ds,
|
||||
hf_tokenizer,
|
||||
num_requests=5,
|
||||
base_items_per_request=0,
|
||||
num_mm_items_range_ratio=0.0,
|
||||
limit_mm_per_prompt={"image": 5, "video": 0},
|
||||
bucket_config={(32, 32, 1): 1.0},
|
||||
)
|
||||
for s in samples:
|
||||
assert s.multi_modal_data == []
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_random_mm_num_items_per_prompt(hf_tokenizer: PreTrainedTokenizerBase) -> None:
|
||||
ds = RandomMultiModalDataset(random_seed=0)
|
||||
# Fixed number of images per prompt
|
||||
# set num_mm_items_range_ratio to 0.0
|
||||
# TODO: modify video values when video sampling is implemented
|
||||
samples_fixed_items = _collect_mm_samples(
|
||||
ds,
|
||||
hf_tokenizer,
|
||||
num_requests=5,
|
||||
base_items_per_request=3,
|
||||
num_mm_items_range_ratio=0.0,
|
||||
limit_mm_per_prompt={"image": 3, "video": 0},
|
||||
bucket_config={(32, 32, 1): 1.0},
|
||||
)
|
||||
# Must have 5 requests each with 3 mm items per prompt
|
||||
assert len(samples_fixed_items) == 5
|
||||
for s in samples_fixed_items:
|
||||
mm_data = cast(list[dict[str, Any]], s.multi_modal_data)
|
||||
assert len(mm_data) == 3
|
||||
for it in mm_data:
|
||||
assert it.get("type") == "image_url"
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_random_mm_bucket_config_not_mutated(
|
||||
hf_tokenizer: PreTrainedTokenizerBase,
|
||||
) -> None:
|
||||
ds = RandomMultiModalDataset(random_seed=0)
|
||||
# This bucket config is not normalized to sum to 1
|
||||
# and has more buckets than requested images
|
||||
original = {(32, 32, 1): 0.2, (52, 64, 1): 6, (25, 64, 1): 3}
|
||||
# Keep a snapshot to compare after sampling
|
||||
snapshot = dict(original)
|
||||
|
||||
_ = _collect_mm_samples(
|
||||
ds,
|
||||
hf_tokenizer,
|
||||
num_requests=4,
|
||||
base_items_per_request=1,
|
||||
num_mm_items_range_ratio=0.0,
|
||||
limit_mm_per_prompt={"image": 1, "video": 0},
|
||||
bucket_config=original,
|
||||
)
|
||||
|
||||
# Ensure the original dict content is unchanged
|
||||
assert original == snapshot
|
||||
|
||||
# Vary number of mm items per prompt
|
||||
# set num_mm_items_range_ratio to 0.5
|
||||
samples_varying_items = _collect_mm_samples(
|
||||
ds,
|
||||
hf_tokenizer,
|
||||
num_requests=5,
|
||||
base_items_per_request=2,
|
||||
num_mm_items_range_ratio=0.5,
|
||||
limit_mm_per_prompt={"image": 4, "video": 0},
|
||||
bucket_config={(32, 32, 1): 1.0},
|
||||
)
|
||||
# Must have 5 requests each with less than 4 mm items per prompt
|
||||
# but at least 1 mm item per prompt
|
||||
assert len(samples_varying_items) == 5
|
||||
for s in samples_varying_items:
|
||||
mm_data = cast(list[dict[str, Any]], s.multi_modal_data)
|
||||
assert len(mm_data) <= 4
|
||||
assert len(mm_data) >= 1
|
||||
for it in mm_data:
|
||||
assert it.get("type") == "image_url"
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_random_mm_video_sampling(hf_tokenizer: PreTrainedTokenizerBase) -> None:
|
||||
"""Test video sampling functionality in RandomMultiModalDataset."""
|
||||
ds = RandomMultiModalDataset(random_seed=42)
|
||||
|
||||
# Test with video bucket configuration
|
||||
bucket_config = {
|
||||
(64, 64, 1): 0.3, # Images
|
||||
(64, 64, 8): 0.7, # Videos
|
||||
}
|
||||
|
||||
limit_mm_per_prompt = {"image": 2, "video": 2}
|
||||
|
||||
samples = _collect_mm_samples(
|
||||
ds,
|
||||
hf_tokenizer,
|
||||
num_requests=5,
|
||||
base_items_per_request=1,
|
||||
num_mm_items_range_ratio=0.0,
|
||||
limit_mm_per_prompt=limit_mm_per_prompt,
|
||||
bucket_config=bucket_config,
|
||||
)
|
||||
|
||||
assert len(samples) == 5
|
||||
|
||||
# Check that we have both images and videos
|
||||
video_count = 0
|
||||
image_count = 0
|
||||
|
||||
for s in samples:
|
||||
mm_data = cast(list[dict[str, Any]], s.multi_modal_data)
|
||||
assert len(mm_data) == 1
|
||||
|
||||
item = mm_data[0]
|
||||
if item.get("type") == "video_url":
|
||||
video_count += 1
|
||||
# Verify video URL format
|
||||
url = item.get("video_url", {}).get("url", "")
|
||||
assert url.startswith("data:video/mp4;base64,")
|
||||
elif item.get("type") == "image_url":
|
||||
image_count += 1
|
||||
# Verify image URL format
|
||||
url = item.get("image_url", {}).get("url", "")
|
||||
assert url.startswith("data:image/jpeg;base64,")
|
||||
|
||||
# Should have some videos due to 0.7 probability
|
||||
assert video_count > 0
|
||||
assert image_count > 0
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_random_mm_video_only_sampling(hf_tokenizer: PreTrainedTokenizerBase) -> None:
|
||||
"""Test sampling with only video buckets."""
|
||||
ds = RandomMultiModalDataset(random_seed=42)
|
||||
|
||||
bucket_config = {
|
||||
(64, 64, 8): 1.0, # Only videos
|
||||
}
|
||||
|
||||
limit_mm_per_prompt = {"image": 0, "video": 1}
|
||||
|
||||
samples = _collect_mm_samples(
|
||||
ds,
|
||||
hf_tokenizer,
|
||||
num_requests=3,
|
||||
base_items_per_request=1,
|
||||
num_mm_items_range_ratio=0.0,
|
||||
limit_mm_per_prompt=limit_mm_per_prompt,
|
||||
bucket_config=bucket_config,
|
||||
)
|
||||
|
||||
assert len(samples) == 3
|
||||
|
||||
for s in samples:
|
||||
mm_data = cast(list[dict[str, Any]], s.multi_modal_data)
|
||||
assert len(mm_data) == 1
|
||||
|
||||
item = mm_data[0]
|
||||
assert item.get("type") == "video_url"
|
||||
url = item.get("video_url", {}).get("url", "")
|
||||
assert url.startswith("data:video/mp4;base64,")
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_random_mm_video_deterministic_sampling(
|
||||
hf_tokenizer: PreTrainedTokenizerBase,
|
||||
) -> None:
|
||||
"""Test that video sampling is deterministic with same seed."""
|
||||
seed = 123
|
||||
ds_a = RandomMultiModalDataset(random_seed=seed)
|
||||
ds_b = RandomMultiModalDataset(random_seed=seed)
|
||||
|
||||
bucket_config = {
|
||||
(64, 64, 8): 1.0, # Only videos
|
||||
}
|
||||
|
||||
limit_mm_per_prompt = {"image": 0, "video": 1}
|
||||
|
||||
a = _collect_mm_samples(
|
||||
ds_a,
|
||||
hf_tokenizer,
|
||||
num_requests=3,
|
||||
base_items_per_request=1,
|
||||
num_mm_items_range_ratio=0.0,
|
||||
limit_mm_per_prompt=limit_mm_per_prompt,
|
||||
bucket_config=bucket_config,
|
||||
)
|
||||
|
||||
b = _collect_mm_samples(
|
||||
ds_b,
|
||||
hf_tokenizer,
|
||||
num_requests=3,
|
||||
base_items_per_request=1,
|
||||
num_mm_items_range_ratio=0.0,
|
||||
limit_mm_per_prompt=limit_mm_per_prompt,
|
||||
bucket_config=bucket_config,
|
||||
)
|
||||
|
||||
fa = [_mm_fingerprint_sample(s) for s in a]
|
||||
fb = [_mm_fingerprint_sample(s) for s in b]
|
||||
assert fa == fb
|
||||
@@ -1,434 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from tempfile import NamedTemporaryFile
|
||||
from typing import Any, cast
|
||||
from unittest.mock import Mock
|
||||
|
||||
import cv2
|
||||
import pybase64 as base64
|
||||
import pytest
|
||||
from transformers import AutoTokenizer, PreTrainedTokenizerBase
|
||||
|
||||
from vllm.benchmarks.datasets import RandomMultiModalDataset, SampleRequest
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def hf_tokenizer() -> PreTrainedTokenizerBase:
|
||||
"""Use a small, commonly available tokenizer."""
|
||||
return AutoTokenizer.from_pretrained("openai-community/gpt2")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def video_dataset() -> RandomMultiModalDataset:
|
||||
"""Create a RandomMultiModalDataset instance for testing."""
|
||||
return RandomMultiModalDataset(random_seed=42)
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_generate_synthetic_video_different_seeds():
|
||||
"""Test that different seeds produce different videos."""
|
||||
dataset1 = RandomMultiModalDataset(random_seed=123)
|
||||
dataset2 = RandomMultiModalDataset(random_seed=456)
|
||||
|
||||
width, height, num_frames = 64, 48, 8
|
||||
|
||||
video1 = dataset1.generate_synthetic_video(width, height, num_frames)
|
||||
video2 = dataset2.generate_synthetic_video(width, height, num_frames)
|
||||
|
||||
# Videos should be different due to different seeds
|
||||
assert video1["bytes"] != video2["bytes"]
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_map_config_to_modality(video_dataset: RandomMultiModalDataset):
|
||||
"""Test modality mapping for different configurations."""
|
||||
# Test image configuration (num_frames = 1)
|
||||
assert video_dataset.map_config_to_modality((256, 256, 1)) == "image"
|
||||
assert video_dataset.map_config_to_modality((720, 1280, 1)) == "image"
|
||||
|
||||
# Test video configurations (num_frames > 1)
|
||||
assert video_dataset.map_config_to_modality((256, 256, 8)) == "video"
|
||||
assert video_dataset.map_config_to_modality((720, 1280, 16)) == "video"
|
||||
assert video_dataset.map_config_to_modality((64, 64, 32)) == "video"
|
||||
|
||||
# Test invalid configurations
|
||||
with pytest.raises(ValueError, match="Invalid multimodal item configuration"):
|
||||
video_dataset.map_config_to_modality((256, 256, 0))
|
||||
|
||||
with pytest.raises(ValueError, match="Invalid multimodal item configuration"):
|
||||
video_dataset.map_config_to_modality((256, 256, -1))
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_generate_mm_item_video(
|
||||
video_dataset: RandomMultiModalDataset,
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""Generate a decodable video without leaving temporary files behind."""
|
||||
monkeypatch.setattr(tempfile, "tempdir", str(tmp_path))
|
||||
# Test video item generation
|
||||
video_config = (64, 48, 8) # height, width, num_frames
|
||||
result = video_dataset.generate_mm_item(video_config)
|
||||
assert not list(tmp_path.iterdir())
|
||||
|
||||
# Check the result structure matches OpenAI API format
|
||||
assert isinstance(result, dict)
|
||||
assert result["type"] == "video_url"
|
||||
assert "video_url" in result
|
||||
assert "url" in result["video_url"]
|
||||
|
||||
# Check that the URL is a data URL with base64 encoded video
|
||||
url = result["video_url"]["url"]
|
||||
assert url.startswith("data:video/mp4;base64,")
|
||||
|
||||
# Decode and verify the video content
|
||||
base64_data = url.split(",")[1]
|
||||
video_bytes = base64.b64decode(base64_data)
|
||||
assert len(video_bytes) > 0
|
||||
|
||||
# Verify the video can be decoded
|
||||
with NamedTemporaryFile(suffix=".mp4", delete=False) as temp_file:
|
||||
temp_path = temp_file.name
|
||||
temp_file.write(video_bytes)
|
||||
|
||||
try:
|
||||
cap = cv2.VideoCapture(temp_path)
|
||||
assert cap.isOpened()
|
||||
|
||||
frame_count = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
|
||||
frame_width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
|
||||
frame_height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
|
||||
|
||||
assert frame_count == 8
|
||||
assert frame_width == 48
|
||||
assert frame_height == 64
|
||||
|
||||
cap.release()
|
||||
finally:
|
||||
if os.path.exists(temp_path):
|
||||
os.unlink(temp_path)
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
@pytest.mark.parametrize("writer_opened", [False, True], ids=["open", "write"])
|
||||
def test_generate_synthetic_video_cleans_up_on_error(
|
||||
video_dataset: RandomMultiModalDataset,
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
writer_opened: bool,
|
||||
):
|
||||
"""Release the writer and remove temporary storage when encoding fails."""
|
||||
monkeypatch.setattr(tempfile, "tempdir", str(tmp_path))
|
||||
writer = Mock(spec=cv2.VideoWriter)
|
||||
writer.isOpened.return_value = writer_opened
|
||||
writer.write.side_effect = RuntimeError("Failed to write video frame")
|
||||
monkeypatch.setattr(cv2, "VideoWriter", lambda *args, **kwargs: writer)
|
||||
|
||||
error = (
|
||||
"Failed to write video frame"
|
||||
if writer_opened
|
||||
else "Failed to create video writer"
|
||||
)
|
||||
with pytest.raises(RuntimeError, match=error):
|
||||
video_dataset.generate_synthetic_video(48, 64, 8)
|
||||
|
||||
assert not list(tmp_path.iterdir())
|
||||
writer.release.assert_called_once_with()
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_generate_mm_item_image(video_dataset: RandomMultiModalDataset):
|
||||
"""Test generating multimodal items for image configurations."""
|
||||
# Test image item generation
|
||||
image_config = (64, 48, 1) # height, width, num_frames=1
|
||||
result = video_dataset.generate_mm_item(image_config)
|
||||
|
||||
# Check the result structure matches OpenAI API format
|
||||
assert isinstance(result, dict)
|
||||
assert result["type"] == "image_url"
|
||||
assert "image_url" in result
|
||||
assert "url" in result["image_url"]
|
||||
|
||||
# Check that the URL is a data URL with base64 encoded image
|
||||
url = result["image_url"]["url"]
|
||||
assert url.startswith("data:image/jpeg;base64,")
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_generate_mm_item_invalid_config(video_dataset: RandomMultiModalDataset):
|
||||
"""Test error handling for invalid configurations."""
|
||||
with pytest.raises(ValueError, match="Invalid multimodal item configuration"):
|
||||
video_dataset.generate_mm_item((256, 256, 0))
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_sample_with_video_buckets(
|
||||
video_dataset: RandomMultiModalDataset, hf_tokenizer: PreTrainedTokenizerBase
|
||||
):
|
||||
"""Test sampling with video bucket configurations."""
|
||||
# Configure bucket with video probability > 0
|
||||
bucket_config = {
|
||||
(64, 64, 1): 0.3, # Images
|
||||
(64, 64, 8): 0.7, # Videos
|
||||
}
|
||||
|
||||
limit_mm_per_prompt = {"image": 5, "video": 3}
|
||||
|
||||
samples = video_dataset.sample(
|
||||
tokenizer=hf_tokenizer,
|
||||
num_requests=5,
|
||||
base_items_per_request=2,
|
||||
num_mm_items_range_ratio=0.0,
|
||||
limit_mm_per_prompt=limit_mm_per_prompt,
|
||||
bucket_config=bucket_config,
|
||||
input_len=20,
|
||||
output_len=5,
|
||||
)
|
||||
|
||||
assert len(samples) == 5
|
||||
|
||||
# Check that samples contain both images and videos
|
||||
video_count = 0
|
||||
image_count = 0
|
||||
|
||||
for sample in samples:
|
||||
assert isinstance(sample, SampleRequest)
|
||||
assert sample.multi_modal_data is not None
|
||||
assert isinstance(sample.multi_modal_data, list)
|
||||
|
||||
mm_data = cast(list[dict[str, Any]], sample.multi_modal_data)
|
||||
assert len(mm_data) == 2 # base_items_per_request
|
||||
|
||||
for item in mm_data:
|
||||
if item["type"] == "video_url":
|
||||
video_count += 1
|
||||
# Verify video URL format
|
||||
url = item["video_url"]["url"]
|
||||
assert url.startswith("data:video/mp4;base64,")
|
||||
elif item["type"] == "image_url":
|
||||
image_count += 1
|
||||
# Verify image URL format
|
||||
url = item["image_url"]["url"]
|
||||
assert url.startswith("data:image/jpeg;base64,")
|
||||
|
||||
# Should have some videos due to 0.7 probability
|
||||
assert video_count > 0
|
||||
assert image_count > 0
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_sample_video_only_buckets(
|
||||
video_dataset: RandomMultiModalDataset, hf_tokenizer: PreTrainedTokenizerBase
|
||||
):
|
||||
"""Test sampling with only video buckets."""
|
||||
bucket_config = {
|
||||
(64, 64, 8): 1.0, # Only videos
|
||||
}
|
||||
|
||||
limit_mm_per_prompt = {"image": 0, "video": 2}
|
||||
|
||||
samples = video_dataset.sample(
|
||||
tokenizer=hf_tokenizer,
|
||||
num_requests=3,
|
||||
base_items_per_request=1,
|
||||
num_mm_items_range_ratio=0.0,
|
||||
limit_mm_per_prompt=limit_mm_per_prompt,
|
||||
bucket_config=bucket_config,
|
||||
input_len=20,
|
||||
output_len=5,
|
||||
)
|
||||
|
||||
assert len(samples) == 3
|
||||
|
||||
for sample in samples:
|
||||
assert isinstance(sample, SampleRequest)
|
||||
assert sample.multi_modal_data is not None
|
||||
assert isinstance(sample.multi_modal_data, list)
|
||||
|
||||
mm_data = cast(list[dict[str, Any]], sample.multi_modal_data)
|
||||
assert len(mm_data) == 1
|
||||
|
||||
item = mm_data[0]
|
||||
assert item["type"] == "video_url"
|
||||
url = item["video_url"]["url"]
|
||||
assert url.startswith("data:video/mp4;base64,")
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_sample_respects_video_limits(
|
||||
video_dataset: RandomMultiModalDataset, hf_tokenizer: PreTrainedTokenizerBase
|
||||
):
|
||||
"""Test that sampling respects video limits per prompt."""
|
||||
bucket_config = {
|
||||
(64, 64, 8): 1.0, # Only videos
|
||||
}
|
||||
|
||||
# Set very low video limit
|
||||
limit_mm_per_prompt = {"image": 0, "video": 1}
|
||||
|
||||
samples = video_dataset.sample(
|
||||
tokenizer=hf_tokenizer,
|
||||
num_requests=3,
|
||||
base_items_per_request=1,
|
||||
num_mm_items_range_ratio=0.0,
|
||||
limit_mm_per_prompt=limit_mm_per_prompt,
|
||||
bucket_config=bucket_config,
|
||||
input_len=20,
|
||||
output_len=5,
|
||||
)
|
||||
|
||||
assert len(samples) == 3
|
||||
|
||||
for sample in samples:
|
||||
mm_data = cast(list[dict[str, Any]], sample.multi_modal_data)
|
||||
assert len(mm_data) <= 1 # Should respect video limit
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_sample_mixed_buckets_with_zero_probability(
|
||||
video_dataset: RandomMultiModalDataset, hf_tokenizer: PreTrainedTokenizerBase
|
||||
):
|
||||
"""Test sampling with mixed buckets including zero probability entries."""
|
||||
bucket_config = {
|
||||
(64, 64, 1): 0.5, # Images
|
||||
(64, 64, 8): 0.5, # Videos
|
||||
(128, 128, 16): 0.0, # Zero probability videos (should be ignored)
|
||||
}
|
||||
|
||||
limit_mm_per_prompt = {"image": 2, "video": 2}
|
||||
|
||||
samples = video_dataset.sample(
|
||||
tokenizer=hf_tokenizer,
|
||||
num_requests=4,
|
||||
base_items_per_request=2,
|
||||
num_mm_items_range_ratio=0.0,
|
||||
limit_mm_per_prompt=limit_mm_per_prompt,
|
||||
bucket_config=bucket_config,
|
||||
input_len=20,
|
||||
output_len=5,
|
||||
)
|
||||
|
||||
assert len(samples) == 4
|
||||
|
||||
# Should only see 64x64 videos, not 128x128 videos
|
||||
for sample in samples:
|
||||
mm_data = cast(list[dict[str, Any]], sample.multi_modal_data)
|
||||
for item in mm_data:
|
||||
if item["type"] == "video_url":
|
||||
# Decode video to verify dimensions
|
||||
url = item["video_url"]["url"]
|
||||
base64_data = url.split(",")[1]
|
||||
video_bytes = base64.b64decode(base64_data)
|
||||
|
||||
with NamedTemporaryFile(suffix=".mp4", delete=False) as temp_file: # noqa
|
||||
temp_path = temp_file.name
|
||||
temp_file.write(video_bytes)
|
||||
|
||||
try:
|
||||
cap = cv2.VideoCapture(temp_path)
|
||||
frame_width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
|
||||
frame_height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
|
||||
cap.release()
|
||||
|
||||
# Should be 64x64, not 128x128
|
||||
assert frame_width == 64
|
||||
assert frame_height == 64
|
||||
finally:
|
||||
if os.path.exists(temp_path):
|
||||
os.unlink(temp_path)
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_sample_deterministic_with_videos(hf_tokenizer: PreTrainedTokenizerBase):
|
||||
"""Test that sampling with videos is deterministic with same seed."""
|
||||
dataset1 = RandomMultiModalDataset(random_seed=123)
|
||||
dataset2 = RandomMultiModalDataset(random_seed=123)
|
||||
|
||||
bucket_config = {
|
||||
(64, 64, 1): 0.3, # Images
|
||||
(64, 64, 8): 0.7, # Videos
|
||||
}
|
||||
|
||||
limit_mm_per_prompt = {"image": 2, "video": 2}
|
||||
|
||||
samples1 = dataset1.sample(
|
||||
tokenizer=hf_tokenizer,
|
||||
num_requests=3,
|
||||
base_items_per_request=1,
|
||||
num_mm_items_range_ratio=0.0,
|
||||
limit_mm_per_prompt=limit_mm_per_prompt,
|
||||
bucket_config=bucket_config,
|
||||
input_len=20,
|
||||
output_len=5,
|
||||
)
|
||||
|
||||
samples2 = dataset2.sample(
|
||||
tokenizer=hf_tokenizer,
|
||||
num_requests=3,
|
||||
base_items_per_request=1,
|
||||
num_mm_items_range_ratio=0.0,
|
||||
limit_mm_per_prompt=limit_mm_per_prompt,
|
||||
bucket_config=bucket_config,
|
||||
input_len=20,
|
||||
output_len=5,
|
||||
)
|
||||
|
||||
assert len(samples1) == len(samples2)
|
||||
|
||||
# Compare multimodal data
|
||||
for s1, s2 in zip(samples1, samples2):
|
||||
assert s1.multi_modal_data == s2.multi_modal_data
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_sample_different_seeds_produce_different_videos(
|
||||
hf_tokenizer: PreTrainedTokenizerBase,
|
||||
):
|
||||
"""Test that different seeds produce different video content."""
|
||||
dataset1 = RandomMultiModalDataset(random_seed=123)
|
||||
dataset2 = RandomMultiModalDataset(random_seed=456)
|
||||
|
||||
bucket_config = {
|
||||
(64, 64, 8): 1.0, # Only videos
|
||||
}
|
||||
|
||||
limit_mm_per_prompt = {"image": 0, "video": 1}
|
||||
|
||||
samples1 = dataset1.sample(
|
||||
tokenizer=hf_tokenizer,
|
||||
num_requests=2,
|
||||
base_items_per_request=1,
|
||||
num_mm_items_range_ratio=0.0,
|
||||
limit_mm_per_prompt=limit_mm_per_prompt,
|
||||
bucket_config=bucket_config,
|
||||
input_len=20,
|
||||
output_len=5,
|
||||
)
|
||||
|
||||
samples2 = dataset2.sample(
|
||||
tokenizer=hf_tokenizer,
|
||||
num_requests=2,
|
||||
base_items_per_request=1,
|
||||
num_mm_items_range_ratio=0.0,
|
||||
limit_mm_per_prompt=limit_mm_per_prompt,
|
||||
bucket_config=bucket_config,
|
||||
input_len=20,
|
||||
output_len=5,
|
||||
)
|
||||
|
||||
# Video content should be different
|
||||
for s1, s2 in zip(samples1, samples2):
|
||||
mm_data1 = cast(list[dict[str, Any]], s1.multi_modal_data)
|
||||
mm_data2 = cast(list[dict[str, Any]], s2.multi_modal_data)
|
||||
|
||||
assert len(mm_data1) == len(mm_data2) == 1
|
||||
|
||||
url1 = mm_data1[0]["video_url"]["url"]
|
||||
url2 = mm_data2[0]["video_url"]["url"]
|
||||
|
||||
assert url1 != url2 # Different video content
|
||||
@@ -1,453 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the ``openai-responses`` backend of ``vllm bench serve``.
|
||||
|
||||
The Responses stream differs from Chat Completions in ways the benchmark client
|
||||
has to get right: every SSE message is labelled with an ``event:`` line,
|
||||
reasoning and output text arrive as separate token-bearing events, and a
|
||||
request ends with ``response.completed`` rather than with its last token. These
|
||||
tests drive the real request function against a local SSE server so the
|
||||
resulting latency and usage numbers are covered end to end.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from collections.abc import Sequence
|
||||
|
||||
import aiohttp
|
||||
import pytest
|
||||
from aiohttp import web
|
||||
from aiohttp.test_utils import TestServer
|
||||
|
||||
from vllm.benchmarks.lib.endpoint_request_func import (
|
||||
ASYNC_REQUEST_FUNCS,
|
||||
OPENAI_COMPATIBLE_BACKENDS,
|
||||
RequestFuncInput,
|
||||
RequestFuncOutput,
|
||||
async_request_openai_responses,
|
||||
)
|
||||
|
||||
# These tests never touch torch, so skip the GPU cleanup fixture.
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
|
||||
MODEL = "openai/gpt-oss-20b"
|
||||
|
||||
# Gaps between scripted events. GAP_TOKEN is much larger than GAP_PREFILL so
|
||||
# that a time to first token measured at the wrong event is unambiguous.
|
||||
GAP_PREFILL = 0.15
|
||||
GAP_TOKEN = 0.5
|
||||
GAP_FINALIZE = 0.3
|
||||
|
||||
|
||||
def sse(event_type: str, payload: dict) -> bytes:
|
||||
"""Frame one event the way ``/v1/responses`` does, with an ``event:`` line.
|
||||
|
||||
``ensure_ascii=False`` matches the server, whose pydantic
|
||||
``model_dump_json`` emits non-ASCII characters unescaped.
|
||||
"""
|
||||
data = json.dumps({"type": event_type, **payload}, ensure_ascii=False)
|
||||
return f"event: {event_type}\ndata: {data}\n\n".encode()
|
||||
|
||||
|
||||
def usage_event(
|
||||
input_tokens: int,
|
||||
output_tokens: int,
|
||||
event_type: str = "response.completed",
|
||||
status: str = "completed",
|
||||
):
|
||||
return sse(
|
||||
event_type,
|
||||
{
|
||||
"response": {
|
||||
"status": status,
|
||||
"usage": {
|
||||
"input_tokens": input_tokens,
|
||||
"output_tokens": output_tokens,
|
||||
"total_tokens": input_tokens + output_tokens,
|
||||
},
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def text_delta(delta: str) -> bytes:
|
||||
return sse("response.output_text.delta", {"delta": delta})
|
||||
|
||||
|
||||
def reasoning_delta(delta: str) -> bytes:
|
||||
return sse("response.reasoning_text.delta", {"delta": delta})
|
||||
|
||||
|
||||
# The metadata events the server sends before any token. None of these may
|
||||
# start the TTFT clock or contribute an ITL sample.
|
||||
PREAMBLE = (
|
||||
sse("response.created", {"response": {"status": "in_progress"}})
|
||||
+ sse("response.in_progress", {"response": {"status": "in_progress"}})
|
||||
+ sse("response.output_item.added", {"output_index": 0})
|
||||
+ sse("response.reasoning_part.added", {"output_index": 0})
|
||||
+ sse("response.content_part.added", {"output_index": 0})
|
||||
)
|
||||
|
||||
# The `.done` family the server sends after the last delta. These repeat the
|
||||
# accumulated text, so timing them would add bogus ITL samples and counting
|
||||
# their payload would duplicate the output.
|
||||
DONE_EVENTS = (
|
||||
sse("response.reasoning_text.done", {"text": "The user asks "})
|
||||
+ sse("response.reasoning_part.done", {"output_index": 0})
|
||||
+ sse("response.output_text.done", {"text": "Rayleigh scattering."})
|
||||
+ sse("response.content_part.done", {"output_index": 0})
|
||||
+ sse("response.output_item.done", {"output_index": 0})
|
||||
)
|
||||
|
||||
|
||||
async def _serve(
|
||||
script: Sequence[bytes | float],
|
||||
*,
|
||||
status: int = 200,
|
||||
chunk_size: int | None = None,
|
||||
seen: dict | None = None,
|
||||
**request_kwargs,
|
||||
) -> RequestFuncOutput:
|
||||
"""Run the request function against a server replaying ``script``.
|
||||
|
||||
Args:
|
||||
script: Byte strings to write, interleaved with floats to sleep for.
|
||||
status: HTTP status to respond with. Non-200 skips the script.
|
||||
chunk_size: If set, split every write into chunks of this many bytes to
|
||||
emulate fragmented transport.
|
||||
seen: If given, populated with the request payload.
|
||||
request_kwargs: Extra fields for the ``RequestFuncInput``.
|
||||
|
||||
"""
|
||||
|
||||
async def handler(request: web.Request) -> web.StreamResponse:
|
||||
if seen is not None:
|
||||
seen["payload"] = await request.json()
|
||||
seen["headers"] = dict(request.headers)
|
||||
if status != 200:
|
||||
return web.Response(status=status, reason="Bad Request")
|
||||
|
||||
response = web.StreamResponse(headers={"Content-Type": "text/event-stream"})
|
||||
await response.prepare(request)
|
||||
for item in script:
|
||||
if isinstance(item, float):
|
||||
await asyncio.sleep(item)
|
||||
elif chunk_size is None:
|
||||
await response.write(item)
|
||||
else:
|
||||
for i in range(0, len(item), chunk_size):
|
||||
await response.write(item[i : i + chunk_size])
|
||||
await response.write_eof()
|
||||
return response
|
||||
|
||||
app = web.Application()
|
||||
app.router.add_post("/v1/responses", handler)
|
||||
server = TestServer(app)
|
||||
await server.start_server()
|
||||
try:
|
||||
request_input = RequestFuncInput(
|
||||
prompt="Why is the sky blue?",
|
||||
api_url=str(server.make_url("/v1/responses")),
|
||||
prompt_len=5,
|
||||
output_len=16,
|
||||
model=MODEL,
|
||||
**request_kwargs,
|
||||
)
|
||||
async with aiohttp.ClientSession() as session:
|
||||
return await async_request_openai_responses(request_input, session)
|
||||
finally:
|
||||
await server.close()
|
||||
|
||||
|
||||
def run(script: Sequence[bytes | float], **kwargs) -> RequestFuncOutput:
|
||||
return asyncio.run(_serve(script, **kwargs))
|
||||
|
||||
|
||||
def test_backend_is_registered_as_openai_compatible():
|
||||
assert ASYNC_REQUEST_FUNCS["openai-responses"] is async_request_openai_responses
|
||||
# Membership enables --ignore-eos on random datasets and the sampling
|
||||
# parameter flags, all of which ResponsesRequest accepts.
|
||||
assert "openai-responses" in OPENAI_COMPATIBLE_BACKENDS
|
||||
|
||||
|
||||
def test_request_payload_is_accepted_by_the_server_schema():
|
||||
"""Every field sent must be a real ResponsesRequest field.
|
||||
|
||||
This is what makes the backend safe to list in OPENAI_COMPATIBLE_BACKENDS:
|
||||
the flags that membership enables have to reach the model, and the server
|
||||
drops unknown fields without complaining.
|
||||
"""
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
|
||||
sampling = {
|
||||
"temperature": 0.7,
|
||||
"top_p": 0.9,
|
||||
"top_k": 20,
|
||||
"min_p": 0.05,
|
||||
"frequency_penalty": 0.2,
|
||||
"presence_penalty": 0.1,
|
||||
"repetition_penalty": 1.05,
|
||||
}
|
||||
seen: dict = {}
|
||||
output = run(
|
||||
[PREAMBLE, text_delta("hi"), usage_event(5, 1)],
|
||||
seen=seen,
|
||||
extra_body=sampling,
|
||||
ignore_eos=True,
|
||||
model_name="served-alias",
|
||||
request_id="bench-7",
|
||||
extra_headers={"x-team": "perf"},
|
||||
)
|
||||
|
||||
assert output.success
|
||||
# --served-model-name wins over --model, as on the other backends.
|
||||
assert seen["payload"]["model"] == "served-alias"
|
||||
# The route is behind validate_json_request, which 415s a missing type.
|
||||
assert seen["headers"]["Content-Type"] == "application/json"
|
||||
assert seen["headers"]["x-request-id"] == "bench-7"
|
||||
assert seen["headers"]["x-team"] == "perf"
|
||||
assert seen["payload"] == {
|
||||
"model": "served-alias",
|
||||
"input": "Why is the sky blue?",
|
||||
"max_output_tokens": 16,
|
||||
"stream": True,
|
||||
"store": False,
|
||||
"ignore_eos": True,
|
||||
**sampling,
|
||||
}
|
||||
assert set(seen["payload"]) <= set(ResponsesRequest.model_fields)
|
||||
ResponsesRequest.model_validate(seen["payload"])
|
||||
|
||||
|
||||
def test_latency_accounting_matches_the_other_endpoints():
|
||||
"""`latency - ttft` must equal `sum(itl)`.
|
||||
|
||||
`serve.py` derives TPOT as `(latency - ttft) / (output_len - 1)`, which only
|
||||
means average inter-token latency when that identity holds. #55508 fixed the
|
||||
same accounting on the chat and audio backends; the trailing events after
|
||||
the last token must not extend the request.
|
||||
"""
|
||||
output = run(
|
||||
[
|
||||
PREAMBLE,
|
||||
GAP_PREFILL / 2,
|
||||
# A keep-alive comment sent during prefill is not a token.
|
||||
b": keep-alive\n\n",
|
||||
GAP_PREFILL / 2,
|
||||
reasoning_delta("The user asks "),
|
||||
GAP_TOKEN,
|
||||
text_delta("Rayleigh"),
|
||||
text_delta(" scattering."),
|
||||
DONE_EVENTS,
|
||||
GAP_FINALIZE,
|
||||
usage_event(5, 8),
|
||||
]
|
||||
)
|
||||
|
||||
assert output.success
|
||||
# Reasoning is timed but not returned, matching how `openai-chat` treats
|
||||
# `DeltaMessage.reasoning`.
|
||||
assert output.generated_text == "Rayleigh scattering."
|
||||
|
||||
# Neither the response.created/in_progress/output_item.added preamble nor
|
||||
# the keep-alive comment may start the clock, and the reasoning delta must,
|
||||
# so TTFT lands on the prefill gap rather than near zero or out at the
|
||||
# first output text delta.
|
||||
assert GAP_PREFILL * 0.8 <= output.ttft < GAP_PREFILL + GAP_TOKEN * 0.5
|
||||
|
||||
# One ITL per token event after the first: the gap before "Rayleigh" and
|
||||
# the back-to-back delta that follows it.
|
||||
assert len(output.itl) == 2
|
||||
assert output.itl[0] >= GAP_TOKEN * 0.5
|
||||
assert output.itl[1] < GAP_TOKEN * 0.5
|
||||
|
||||
# The identity is exact, so no wall-clock margin is involved. The .done
|
||||
# family and the GAP_FINALIZE wait before response.completed sit outside
|
||||
# the request, exactly as the trailing usage chunk does on chat.
|
||||
assert output.latency - output.ttft == pytest.approx(sum(output.itl), abs=1e-9)
|
||||
|
||||
assert output.start_time > 0.0
|
||||
|
||||
|
||||
def test_usage_is_taken_from_the_terminal_event():
|
||||
output = run([PREAMBLE, text_delta("blue"), usage_event(17, 4)])
|
||||
|
||||
assert output.success
|
||||
assert output.prompt_len == 17
|
||||
assert output.output_tokens == 4
|
||||
|
||||
|
||||
@pytest.mark.parametrize("event_type", ["response.completed", "response.incomplete"])
|
||||
def test_incomplete_response_is_a_successful_request(event_type: str):
|
||||
"""Hitting max_output_tokens is the Responses form of finish_reason=length.
|
||||
|
||||
vLLM reports it as `response.completed` carrying `status: "incomplete"`.
|
||||
The spec's dedicated `response.incomplete` event is accepted as well.
|
||||
"""
|
||||
output = run(
|
||||
[
|
||||
PREAMBLE,
|
||||
text_delta("truncated"),
|
||||
usage_event(17, 16, event_type=event_type, status="incomplete"),
|
||||
]
|
||||
)
|
||||
|
||||
assert output.success
|
||||
assert output.generated_text == "truncated"
|
||||
assert output.output_tokens == 16
|
||||
|
||||
|
||||
@pytest.mark.parametrize("chunk_size", [1, 3, 7, 64])
|
||||
def test_output_survives_transport_chunk_boundaries(chunk_size: int):
|
||||
"""Splits inside the event line, the JSON, or a multibyte character."""
|
||||
text = " leading 中 trailing "
|
||||
output = run(
|
||||
[PREAMBLE, text_delta(text), usage_event(5, 6)],
|
||||
chunk_size=chunk_size,
|
||||
)
|
||||
|
||||
assert output.success
|
||||
assert output.generated_text == text
|
||||
assert output.output_tokens == 6
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"char", ["\u2028", "\u2029", "\u0085"], ids=["u2028", "u2029", "u0085"]
|
||||
)
|
||||
def test_unicode_line_separators_do_not_truncate_the_payload(char: str):
|
||||
"""The server emits these unescaped and they are legal inside a JSON string.
|
||||
|
||||
`str.splitlines()` treats them as line breaks, which would cut the `data:`
|
||||
line short and fail the request on valid model output.
|
||||
"""
|
||||
text = f"before{char}after"
|
||||
output = run([PREAMBLE, text_delta(text), usage_event(5, 3)])
|
||||
|
||||
assert output.success, output.error
|
||||
assert output.generated_text == text
|
||||
|
||||
|
||||
def test_empty_data_line_is_skipped():
|
||||
"""A bare `data:` line carries no JSON and must not fail the request."""
|
||||
output = run([PREAMBLE, b"data:\n\n", text_delta("ok"), usage_event(5, 1)])
|
||||
|
||||
assert output.success, output.error
|
||||
assert output.generated_text == "ok"
|
||||
|
||||
|
||||
def test_prompt_len_falls_back_when_usage_is_absent():
|
||||
"""A terminal event without usage keeps the locally computed prompt length."""
|
||||
output = run(
|
||||
[
|
||||
PREAMBLE,
|
||||
text_delta("hi"),
|
||||
sse("response.completed", {"response": {"status": "completed"}}),
|
||||
]
|
||||
)
|
||||
|
||||
assert output.success, output.error
|
||||
assert output.prompt_len == 5
|
||||
|
||||
|
||||
def test_multi_line_data_payload_is_joined_with_newlines():
|
||||
"""The SSE spec allows one payload to span several `data:` lines."""
|
||||
script: list[bytes | float] = [
|
||||
PREAMBLE,
|
||||
b"event: response.output_text.delta\n"
|
||||
b'data: {"type": "response.output_text.delta",\n'
|
||||
b'data: "delta": "ok"}\n\n',
|
||||
usage_event(5, 1),
|
||||
]
|
||||
output = run(script)
|
||||
|
||||
assert output.success, output.error
|
||||
assert output.generated_text == "ok"
|
||||
|
||||
|
||||
def test_done_sentinel_is_skipped():
|
||||
"""Other Responses servers send `[DONE]`, but vLLM does not."""
|
||||
output = run([PREAMBLE, text_delta("ok"), usage_event(5, 1), b"data: [DONE]\n\n"])
|
||||
|
||||
assert output.success, output.error
|
||||
assert output.generated_text == "ok"
|
||||
|
||||
|
||||
def test_error_before_any_token_reports_the_server_error():
|
||||
"""The server's message must win over the generic no-TTFT message."""
|
||||
output = run(
|
||||
[PREAMBLE, b'event: error\ndata: {"type": "error", "message": "boom"}\n\n']
|
||||
)
|
||||
|
||||
assert not output.success
|
||||
assert "boom" in output.error
|
||||
|
||||
|
||||
def test_http_error_is_not_reported_as_success():
|
||||
output = run([], status=400)
|
||||
|
||||
assert not output.success
|
||||
assert output.error
|
||||
|
||||
|
||||
def test_stream_without_tokens_is_not_reported_as_success():
|
||||
"""A stream that finishes without generating anything has no TTFT."""
|
||||
output = run([PREAMBLE, usage_event(5, 0)])
|
||||
|
||||
assert not output.success
|
||||
assert "TTFT" in output.error
|
||||
|
||||
|
||||
def test_truncated_stream_is_not_reported_as_success():
|
||||
"""A stream cut short after a token has no usage and an understated E2EL."""
|
||||
output = run([PREAMBLE, text_delta("partial")])
|
||||
|
||||
assert not output.success
|
||||
assert "terminal event" in output.error
|
||||
|
||||
|
||||
def test_mid_stream_error_is_not_reported_as_success():
|
||||
"""An error payload mid-stream must not be counted as a success.
|
||||
|
||||
vLLM cannot emit this today: its streaming error path fails to validate
|
||||
against the event union, so a generation failure truncates the stream
|
||||
instead. Proxies and other Responses servers do send it.
|
||||
"""
|
||||
output = run(
|
||||
[
|
||||
PREAMBLE,
|
||||
text_delta("partial"),
|
||||
b'event: error\ndata: {"error": {"message": "boom", "code": 500}}\n\n',
|
||||
]
|
||||
)
|
||||
|
||||
assert not output.success
|
||||
assert "boom" in output.error
|
||||
|
||||
|
||||
def test_data_only_framing_is_still_parsed():
|
||||
"""Some proxies drop the ``event:`` line and forward only ``data:``."""
|
||||
script: list[bytes | float] = [
|
||||
b'data: {"type": "response.output_text.delta", "delta": "ok"}\n\n',
|
||||
usage_event(5, 1),
|
||||
]
|
||||
output = run(script)
|
||||
|
||||
assert output.success
|
||||
assert output.generated_text == "ok"
|
||||
|
||||
|
||||
def test_endpoint_path_is_validated():
|
||||
request_input = RequestFuncInput(
|
||||
prompt="hi",
|
||||
api_url="http://localhost:8000/v1/chat/completions",
|
||||
prompt_len=1,
|
||||
output_len=1,
|
||||
model=MODEL,
|
||||
)
|
||||
|
||||
async def call() -> None:
|
||||
async with aiohttp.ClientSession() as session:
|
||||
await async_request_openai_responses(request_input, session)
|
||||
|
||||
with pytest.raises(ValueError, match="responses"):
|
||||
asyncio.run(call())
|
||||
@@ -1,61 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Keep the `vllm bench serve` flag snapshot in sync for the Rust CLI parity test.
|
||||
|
||||
`rust/src/bench/tests/cli_parity.rs` asserts that every flag in the snapshot is
|
||||
either a known `vllm-bench` clap flag/alias or explicitly allowlisted as
|
||||
Python-only, so `VLLM_USE_RUST_BENCH=1` delegation cannot silently reject
|
||||
documented flags. This test keeps the snapshot itself current.
|
||||
|
||||
Run this file directly to regenerate the snapshot:
|
||||
|
||||
.venv/bin/python tests/benchmarks/test_rust_bench_cli_parity.py
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
SNAPSHOT = REPO_ROOT / "rust" / "src" / "bench" / "tests" / "python_serve_flags.txt"
|
||||
HEADER = """\
|
||||
# Long CLI flags of the Python `vllm bench serve` parser
|
||||
# (vllm.benchmarks.serve.add_cli_args), sorted; --help excluded.
|
||||
# Regenerate: .venv/bin/python tests/benchmarks/test_rust_bench_cli_parity.py
|
||||
# Consumed by rust/src/bench/tests/cli_parity.rs, which requires every flag
|
||||
# below to be a known vllm-bench flag/alias or a PYTHON_ONLY allowlist entry.
|
||||
"""
|
||||
|
||||
|
||||
def _current_flags() -> list[str]:
|
||||
from vllm.benchmarks.serve import add_cli_args
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
parser = FlexibleArgumentParser()
|
||||
add_cli_args(parser)
|
||||
return sorted(
|
||||
{
|
||||
opt
|
||||
for action in parser._actions
|
||||
for opt in action.option_strings
|
||||
if opt.startswith("--") and opt != "--help"
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _snapshot_flags() -> list[str]:
|
||||
lines = SNAPSHOT.read_text().splitlines()
|
||||
return [line.strip() for line in lines if line.strip() and not line.startswith("#")]
|
||||
|
||||
|
||||
def test_serve_flag_snapshot_is_current():
|
||||
assert SNAPSHOT.is_file(), f"missing snapshot {SNAPSHOT}"
|
||||
assert _current_flags() == _snapshot_flags(), (
|
||||
"`vllm bench serve` flags changed. Regenerate the snapshot with "
|
||||
"`.venv/bin/python tests/benchmarks/test_rust_bench_cli_parity.py`, "
|
||||
"then for any added flag either support it in rust/src/bench/src/cli.rs "
|
||||
"or add it to PYTHON_ONLY in rust/src/bench/tests/cli_parity.rs"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
SNAPSHOT.write_text(HEADER + "\n".join(_current_flags()) + "\n")
|
||||
print(f"wrote {SNAPSHOT}")
|
||||
@@ -1,258 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm.benchmarks.datasets.utils import get_sampling_params
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
|
||||
|
||||
class _FakeTokenizer(TokenizerLike):
|
||||
"""Minimal tokenizer implementing the TokenizerLike protocol
|
||||
for testing get_sampling_params."""
|
||||
|
||||
def __init__(self, vocab_size: int = 1000, num_special_tokens: int = 0) -> None:
|
||||
self._vocab_size = vocab_size
|
||||
self._num_special_tokens = num_special_tokens
|
||||
|
||||
# -- Properties required by TokenizerLike --
|
||||
|
||||
@classmethod
|
||||
def from_pretrained(cls, path_or_repo_id, *a, **kw):
|
||||
return cls()
|
||||
|
||||
@property
|
||||
def vocab_size(self) -> int:
|
||||
return self._vocab_size
|
||||
|
||||
@property
|
||||
def all_special_tokens(self) -> list[str]:
|
||||
return []
|
||||
|
||||
@property
|
||||
def all_special_ids(self) -> list[int]:
|
||||
return []
|
||||
|
||||
@property
|
||||
def bos_token_id(self) -> int:
|
||||
return 0
|
||||
|
||||
@property
|
||||
def eos_token_id(self) -> int:
|
||||
return 1
|
||||
|
||||
@property
|
||||
def pad_token_id(self) -> int:
|
||||
return 2
|
||||
|
||||
@property
|
||||
def is_fast(self) -> bool:
|
||||
return False
|
||||
|
||||
@property
|
||||
def max_token_id(self) -> int:
|
||||
return self._vocab_size - 1
|
||||
|
||||
@property
|
||||
def max_chars_per_token(self) -> int:
|
||||
return 4
|
||||
|
||||
@property
|
||||
def truncation_side(self) -> str:
|
||||
return "right"
|
||||
|
||||
def num_special_tokens_to_add(self) -> int:
|
||||
return self._num_special_tokens
|
||||
|
||||
def __call__(self, text, text_pair=None, **kw): # type: ignore[override]
|
||||
raise NotImplementedError
|
||||
|
||||
def get_vocab(self) -> dict[str, int]:
|
||||
return {}
|
||||
|
||||
def get_added_vocab(self) -> dict[str, int]:
|
||||
return {}
|
||||
|
||||
def encode(self, text, **kw) -> list[int]: # type: ignore[override]
|
||||
raise NotImplementedError
|
||||
|
||||
def apply_chat_template(self, messages, **kw): # type: ignore[override]
|
||||
raise NotImplementedError
|
||||
|
||||
def convert_tokens_to_ids(self, tokens):
|
||||
raise NotImplementedError
|
||||
|
||||
def convert_tokens_to_string(self, tokens: list[str]) -> str:
|
||||
raise NotImplementedError
|
||||
|
||||
def decode(self, ids, skip_special_tokens: bool = False) -> str:
|
||||
raise NotImplementedError
|
||||
|
||||
def convert_ids_to_tokens(
|
||||
self, ids, skip_special_tokens: bool = False
|
||||
) -> list[str]:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class TestGetSamplingParams:
|
||||
"""Tests for ``get_sampling_params`` in ``vllm.benchmarks.datasets.shared``."""
|
||||
|
||||
# -- helpers --
|
||||
|
||||
@staticmethod
|
||||
def _tok(vocab_size: int = 1000, num_special: int = 0) -> _FakeTokenizer:
|
||||
return _FakeTokenizer(vocab_size=vocab_size, num_special_tokens=num_special)
|
||||
|
||||
# -- return shape / dtype --
|
||||
|
||||
def test_returns_three_arrays(self):
|
||||
rng = np.random.default_rng(0)
|
||||
result = get_sampling_params(rng, 5, 0.0, 100, 50, self._tok())
|
||||
assert len(result) == 3
|
||||
for arr in result:
|
||||
assert isinstance(arr, np.ndarray)
|
||||
|
||||
@pytest.mark.parametrize("n", [1, 10, 100])
|
||||
def test_output_length_matches_num_requests(self, n: int):
|
||||
rng = np.random.default_rng(42)
|
||||
input_lens, output_lens, offsets = get_sampling_params(
|
||||
rng, n, 0.0, 64, 32, self._tok()
|
||||
)
|
||||
assert input_lens.shape == (n,)
|
||||
assert output_lens.shape == (n,)
|
||||
assert offsets.shape == (n,)
|
||||
|
||||
# -- fixed lengths (range_ratio = 0) --
|
||||
|
||||
def test_zero_range_ratio_gives_constant_lengths(self):
|
||||
rng = np.random.default_rng(7)
|
||||
input_lens, output_lens, _ = get_sampling_params(
|
||||
rng, 20, 0.0, 128, 64, self._tok()
|
||||
)
|
||||
assert np.all(input_lens == 128)
|
||||
assert np.all(output_lens == 64)
|
||||
|
||||
def test_special_tokens_subtracted_from_input_only(self):
|
||||
rng = np.random.default_rng(7)
|
||||
input_lens, output_lens, _ = get_sampling_params(
|
||||
rng, 10, 0.0, 100, 50, self._tok(num_special=4)
|
||||
)
|
||||
# real_input_len = 100 - 4 = 96, range_ratio 0 → all 96
|
||||
assert np.all(input_lens == 96)
|
||||
# special tokens are not subtracted from output length
|
||||
assert np.all(output_lens == 50)
|
||||
|
||||
# -- range ratios --
|
||||
|
||||
def test_input_range_bounds(self):
|
||||
rng = np.random.default_rng(0)
|
||||
ratio = 0.5
|
||||
base = 200
|
||||
input_lens, _, _ = get_sampling_params(
|
||||
rng, 500, {"input": ratio, "output": 0.0}, base, 50, self._tok()
|
||||
)
|
||||
lo = int(np.floor(base * (1 - ratio)))
|
||||
hi = int(np.ceil(base * (1 + ratio)))
|
||||
assert np.all(input_lens >= lo)
|
||||
assert np.all(input_lens <= hi)
|
||||
|
||||
def test_output_range_bounds(self):
|
||||
rng = np.random.default_rng(0)
|
||||
ratio = 0.3
|
||||
base = 100
|
||||
_, output_lens, _ = get_sampling_params(
|
||||
rng, 500, {"input": 0.0, "output": ratio}, 50, base, self._tok()
|
||||
)
|
||||
lo = max(1, int(np.floor(base * (1 - ratio))))
|
||||
hi = int(np.ceil(base * (1 + ratio)))
|
||||
assert np.all(output_lens >= lo)
|
||||
assert np.all(output_lens <= hi)
|
||||
|
||||
def test_output_low_clamped_to_one(self):
|
||||
"""Even with a high ratio that would push output_low to 0,
|
||||
the function clamps it to 1."""
|
||||
rng = np.random.default_rng(0)
|
||||
# output_len=1, ratio=0.99 → floor(1*0.01)=0, should clamp to 1
|
||||
_, output_lens, _ = get_sampling_params(
|
||||
rng, 50, {"input": 0.0, "output": 0.99}, 100, 1, self._tok()
|
||||
)
|
||||
assert np.all(output_lens >= 1)
|
||||
|
||||
# -- offsets bounded by vocab_size --
|
||||
|
||||
@pytest.mark.parametrize("vocab", [100, 32000, 128256])
|
||||
def test_offsets_within_vocab(self, vocab: int):
|
||||
rng = np.random.default_rng(0)
|
||||
_, _, offsets = get_sampling_params(
|
||||
rng, 200, 0.0, 64, 32, self._tok(vocab_size=vocab)
|
||||
)
|
||||
assert np.all(offsets >= 0)
|
||||
assert np.all(offsets < vocab)
|
||||
|
||||
# -- reproducibility --
|
||||
|
||||
def test_same_seed_same_results(self):
|
||||
tok = self._tok()
|
||||
rr = {"input": 0.3, "output": 0.2}
|
||||
a = get_sampling_params(np.random.default_rng(42), 50, rr, 256, 64, tok)
|
||||
b = get_sampling_params(np.random.default_rng(42), 50, rr, 256, 64, tok)
|
||||
for arr_a, arr_b in zip(a, b):
|
||||
np.testing.assert_array_equal(arr_a, arr_b)
|
||||
|
||||
def test_different_seed_different_results(self):
|
||||
tok = self._tok()
|
||||
rr = {"input": 0.3, "output": 0.2}
|
||||
a = get_sampling_params(np.random.default_rng(0), 50, rr, 256, 64, tok)
|
||||
b = get_sampling_params(np.random.default_rng(1), 50, rr, 256, 64, tok)
|
||||
# Extremely unlikely all three arrays match with different seeds
|
||||
assert not all(np.array_equal(arr_a, arr_b) for arr_a, arr_b in zip(a, b))
|
||||
|
||||
# -- validation / error paths --
|
||||
|
||||
@pytest.mark.parametrize("bad_ratio", [-0.1, 1.0, 1.5])
|
||||
def test_invalid_input_range_ratio(self, bad_ratio: float):
|
||||
rng = np.random.default_rng(0)
|
||||
with pytest.raises(ValueError, match="input_range_ratio"):
|
||||
get_sampling_params(
|
||||
rng, 10, {"input": bad_ratio, "output": 0.0}, 100, 50, self._tok()
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize("bad_ratio", [-0.1, 1.0, 1.5])
|
||||
def test_invalid_output_range_ratio(self, bad_ratio: float):
|
||||
rng = np.random.default_rng(0)
|
||||
with pytest.raises(ValueError, match="output_range_ratio"):
|
||||
get_sampling_params(
|
||||
rng, 10, {"input": 0.0, "output": bad_ratio}, 100, 50, self._tok()
|
||||
)
|
||||
|
||||
def test_invalid_dict_missing_keys(self):
|
||||
rng = np.random.default_rng(0)
|
||||
with pytest.raises(ValueError, match="input.*output"):
|
||||
get_sampling_params(rng, 10, {"input": 0.1}, 100, 50, self._tok())
|
||||
|
||||
def test_input_len_zero_with_special_tokens(self):
|
||||
"""input_len < num_special_tokens → real_input_len = 0, which is fine
|
||||
(range [0, 0])."""
|
||||
rng = np.random.default_rng(0)
|
||||
input_lens, _, _ = get_sampling_params(
|
||||
rng, 5, 0.0, 5, 50, self._tok(num_special=10)
|
||||
)
|
||||
# real_input_len = max(0, 5 - 10) = 0
|
||||
assert np.all(input_lens == 0)
|
||||
|
||||
# -- edge cases --
|
||||
|
||||
def test_single_request(self):
|
||||
rng = np.random.default_rng(0)
|
||||
i, o, off = get_sampling_params(rng, 1, 0.0, 100, 50, self._tok())
|
||||
assert i.shape == (1,)
|
||||
assert o.shape == (1,)
|
||||
assert off.shape == (1,)
|
||||
|
||||
def test_large_num_requests(self):
|
||||
rng = np.random.default_rng(0)
|
||||
i, o, off = get_sampling_params(rng, 10_000, 0.5, 512, 128, self._tok())
|
||||
assert i.shape == (10_000,)
|
||||
assert o.shape == (10_000,)
|
||||
assert off.shape == (10_000,)
|
||||
@@ -1,163 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Regression test for --skip-tokenizer-init with --dataset-name custom.
|
||||
|
||||
Before the fix (introduced by #39896), running:
|
||||
|
||||
vllm bench serve \
|
||||
--backend vllm-pooling \
|
||||
--dataset-name custom \
|
||||
--dataset-path <path> \
|
||||
--model ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11 \
|
||||
--endpoint /pooling \
|
||||
--skip-tokenizer-init \
|
||||
...
|
||||
|
||||
raised immediately with:
|
||||
|
||||
AssertionError: Tokenizer must be initialized before loading dataset
|
||||
|
||||
even though CustomDataset.sample() already handles tokenizer=None.
|
||||
This test exercises main_async() directly so it catches any regression
|
||||
re-introduced at the serve.py level, not just inside get_samples().
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
from pathlib import Path
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
import vllm.benchmarks.serve as serve_module
|
||||
|
||||
# Exact prompt payload from the failing benchmark run against a
|
||||
# Prithvi-EO-2.0 pooling endpoint (URL-in / base64-out format).
|
||||
_PRITHVI_PROMPT = {
|
||||
"data": {
|
||||
"data": "https://huggingface.co/christian-pinto/Prithvi-EO-2.0-300M-TL-VLLM/resolve/main/India_900498_S2Hand.tif",
|
||||
"data_format": "url",
|
||||
"out_data_format": "b64_json",
|
||||
"indices": [1, 2, 3, 8, 11, 12],
|
||||
},
|
||||
"priority": 0,
|
||||
"softmax": False,
|
||||
}
|
||||
|
||||
|
||||
def _write_dataset(path: Path) -> None:
|
||||
path.write_text(json.dumps({"prompt": _PRITHVI_PROMPT}) + "\n")
|
||||
|
||||
|
||||
def _args(dataset_path: str) -> argparse.Namespace:
|
||||
"""Reproduce the argparse.Namespace that serve.py builds from the
|
||||
failing command, including skip_tokenizer_init=True."""
|
||||
return argparse.Namespace(
|
||||
# dataset
|
||||
dataset_name="custom",
|
||||
dataset_path=dataset_path,
|
||||
disable_shuffle=False,
|
||||
num_prompts=1,
|
||||
custom_output_len=256,
|
||||
skip_chat_template=True,
|
||||
chat_template_kwargs=None,
|
||||
no_oversample=False,
|
||||
seed=0,
|
||||
request_id_prefix="bench-",
|
||||
# model / tokenizer
|
||||
model="ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11",
|
||||
served_model_name=None,
|
||||
tokenizer=None,
|
||||
tokenizer_mode="auto",
|
||||
trust_remote_code=False,
|
||||
skip_tokenizer_init=True, # <-- the flag under test
|
||||
# backend / endpoint
|
||||
backend="vllm-pooling",
|
||||
base_url="http://127.0.0.1:8000",
|
||||
host="127.0.0.1",
|
||||
port=8000,
|
||||
endpoint="/pooling",
|
||||
header=None,
|
||||
insecure=False,
|
||||
# traffic
|
||||
request_rate=16.0,
|
||||
burstiness=1.0,
|
||||
max_concurrency=None,
|
||||
probe_request_rate=0.0,
|
||||
# misc serve args that main_async reads before reaching get_samples
|
||||
plot_timeline=False,
|
||||
plot_dataset_stats=False,
|
||||
self_timed=None,
|
||||
metadata=None,
|
||||
label=None,
|
||||
logprobs=None,
|
||||
use_beam_search=False,
|
||||
ignore_eos=False,
|
||||
goodput=None,
|
||||
percentile_metrics="ttft,tpot,itl,e2el",
|
||||
metric_percentiles="25,50,75,99",
|
||||
save_result=False,
|
||||
append_result=False,
|
||||
result_dir=".",
|
||||
result_filename=None,
|
||||
num_warmups=0,
|
||||
profile=False,
|
||||
disable_tqdm=True,
|
||||
lora_modules=None,
|
||||
lora_assignment="random",
|
||||
ramp_up_strategy=None,
|
||||
ramp_up_start_rps=None,
|
||||
ramp_up_end_rps=None,
|
||||
ready_check_timeout_sec=0,
|
||||
extra_body=None,
|
||||
top_p=None,
|
||||
top_k=None,
|
||||
min_p=None,
|
||||
temperature=None,
|
||||
frequency_penalty=None,
|
||||
presence_penalty=None,
|
||||
repetition_penalty=None,
|
||||
save_detailed=False,
|
||||
input_len=None,
|
||||
output_len=None,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_main_async_skip_tokenizer_init_does_not_raise(tmp_path: Path) -> None:
|
||||
"""main_async must not raise AssertionError when skip_tokenizer_init=True
|
||||
and dataset_name='custom'.
|
||||
|
||||
On main (before the fix) this test fails with:
|
||||
AssertionError: Tokenizer must be initialized before loading dataset
|
||||
"""
|
||||
dataset_path = tmp_path / "dataset_url_input_india.jsonl"
|
||||
_write_dataset(dataset_path)
|
||||
|
||||
args = _args(str(dataset_path))
|
||||
|
||||
# Patch benchmark() so we never make real HTTP requests — the regression
|
||||
# triggers before benchmark() is ever called, so this just keeps the test
|
||||
# fast and hermetic.
|
||||
mock_result = {
|
||||
"completed": 1,
|
||||
"failed": 0,
|
||||
"total_input_tokens": 1,
|
||||
"total_output_tokens": 1,
|
||||
"request_throughput": 1.0,
|
||||
"output_throughput": 1.0,
|
||||
"total_token_throughput": 1.0,
|
||||
"input_lens": [],
|
||||
"output_lens": [],
|
||||
"ttfts": [],
|
||||
"itls": [],
|
||||
"generated_texts": [],
|
||||
"errors": [],
|
||||
"duration": 1.0,
|
||||
}
|
||||
with patch.object(
|
||||
serve_module, "benchmark", new=AsyncMock(return_value=mock_result)
|
||||
):
|
||||
# Must NOT raise AssertionError
|
||||
asyncio.run(serve_module.main_async(args))
|
||||
@@ -1,369 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for ``vllm bench throughput`` CLI.
|
||||
|
||||
Throughput reuses ``bench serve``'s ``datasets.get_samples`` dispatch; the
|
||||
LoRA-assignment and MMVU cases below cover the pieces that are throughput-only
|
||||
(serve has no analogue), keeping the two benchmarks aligned without duplicating
|
||||
serve-side dataset coverage.
|
||||
"""
|
||||
|
||||
import subprocess
|
||||
from argparse import Namespace
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from transformers import AutoTokenizer, PreTrainedTokenizerBase
|
||||
|
||||
from vllm.benchmarks.datasets import SampleRequest
|
||||
from vllm.benchmarks.throughput import (
|
||||
_run_vllm_chat_requests,
|
||||
_to_serve_args,
|
||||
add_cli_args,
|
||||
assign_loras,
|
||||
get_requests,
|
||||
)
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
MODEL_NAME = "meta-llama/Llama-3.2-1B-Instruct"
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_bench_throughput():
|
||||
command = [
|
||||
"vllm",
|
||||
"bench",
|
||||
"throughput",
|
||||
"--model",
|
||||
MODEL_NAME,
|
||||
"--input-len",
|
||||
"32",
|
||||
"--output-len",
|
||||
"1",
|
||||
"--enforce-eager",
|
||||
"--load-format",
|
||||
"dummy",
|
||||
]
|
||||
result = subprocess.run(command, capture_output=True, text=True)
|
||||
print(result.stdout)
|
||||
print(result.stderr)
|
||||
|
||||
assert result.returncode == 0, f"Benchmark failed: {result.stderr}"
|
||||
|
||||
|
||||
def test_bench_throughput_accepts_custom_audio_args():
|
||||
parser = FlexibleArgumentParser()
|
||||
add_cli_args(parser)
|
||||
|
||||
args = parser.parse_args(
|
||||
[
|
||||
"--dataset-name",
|
||||
"custom_audio",
|
||||
"--dataset-path",
|
||||
"audio.jsonl",
|
||||
"--no-oversample",
|
||||
"--custom-output-len",
|
||||
"32",
|
||||
"--enable-multimodal-chat",
|
||||
]
|
||||
)
|
||||
|
||||
assert args.dataset_name == "custom_audio"
|
||||
assert args.no_oversample
|
||||
assert args.custom_output_len == 32
|
||||
assert args.enable_multimodal_chat
|
||||
|
||||
|
||||
def test_vllm_chat_requests_include_multimodal_content():
|
||||
class FakeLLM:
|
||||
def __init__(self):
|
||||
self.prompts = None
|
||||
|
||||
def chat(self, prompts, sampling_params, use_tqdm):
|
||||
del sampling_params, use_tqdm
|
||||
self.prompts = prompts
|
||||
return []
|
||||
|
||||
llm = FakeLLM()
|
||||
audio_content = {
|
||||
"type": "input_audio",
|
||||
"input_audio": {"data": "abc", "format": "wav"},
|
||||
}
|
||||
request = SampleRequest(
|
||||
prompt="Transcribe this audio.",
|
||||
prompt_len=1,
|
||||
expected_output_len=8,
|
||||
multi_modal_data=audio_content,
|
||||
)
|
||||
|
||||
_run_vllm_chat_requests(
|
||||
llm,
|
||||
[request],
|
||||
n=1,
|
||||
disable_detokenize=False,
|
||||
do_profile=False,
|
||||
prequeue_requests=False,
|
||||
)
|
||||
|
||||
assert llm.prompts == [
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Transcribe this audio."},
|
||||
audio_content,
|
||||
],
|
||||
}
|
||||
]
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def hf_tokenizer() -> PreTrainedTokenizerBase:
|
||||
# Small, commonly available tokenizer.
|
||||
return AutoTokenizer.from_pretrained("openai-community/gpt2")
|
||||
|
||||
|
||||
# -----------------------------
|
||||
# LoRA assignment (throughput-only post-processing)
|
||||
# -----------------------------
|
||||
|
||||
|
||||
def _sr(i: int) -> SampleRequest:
|
||||
return SampleRequest(prompt=f"prompt {i}", prompt_len=4, expected_output_len=2)
|
||||
|
||||
|
||||
def _lora_args(
|
||||
*,
|
||||
lora_path: str | None,
|
||||
max_loras: int,
|
||||
lora_assignment: str = "random",
|
||||
) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
lora_path=lora_path,
|
||||
max_loras=max_loras,
|
||||
lora_assignment=lora_assignment,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_assign_loras_noop_without_lora_path() -> None:
|
||||
"""No lora_path -> requests returned unchanged, no LoRA attached."""
|
||||
reqs = [_sr(i) for i in range(5)]
|
||||
out = assign_loras(reqs, _lora_args(lora_path=None, max_loras=4))
|
||||
assert out is reqs
|
||||
assert all(r.lora_request is None for r in out)
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_assign_loras_round_robin() -> None:
|
||||
"""Round-robin assigns deterministic index % max_loras + 1 IDs."""
|
||||
reqs = [_sr(i) for i in range(6)]
|
||||
out = assign_loras(
|
||||
reqs,
|
||||
_lora_args(lora_path="/tmp/lora", max_loras=3, lora_assignment="round-robin"),
|
||||
)
|
||||
assert [r.lora_request.lora_int_id for r in out] == [1, 2, 3, 1, 2, 3]
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_assign_loras_random_in_range() -> None:
|
||||
"""Random assignment stays in [1, max_loras] and covers all IDs."""
|
||||
reqs = [_sr(i) for i in range(200)]
|
||||
out = assign_loras(
|
||||
reqs,
|
||||
_lora_args(lora_path="/tmp/lora", max_loras=5, lora_assignment="random"),
|
||||
)
|
||||
ids = [r.lora_request.lora_int_id for r in out]
|
||||
assert all(1 <= i <= 5 for i in ids)
|
||||
assert len(set(ids)) == 5
|
||||
|
||||
|
||||
# -----------------------------
|
||||
# Issue #50838: MMVU via throughput
|
||||
# -----------------------------
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_get_requests_resolves_mmvu(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
hf_tokenizer: PreTrainedTokenizerBase,
|
||||
) -> None:
|
||||
"""Issue #50838: --dataset-path yale-nlp/MMVU is accepted by throughput.
|
||||
|
||||
Previously throughput's validate_args rejected MMVU ("is not supported by
|
||||
hf dataset"). It now resolves through the shared get_samples. The HF
|
||||
download is stubbed so the resolution + dispatch path is exercised without
|
||||
network access.
|
||||
"""
|
||||
import vllm.benchmarks.datasets.datasets as dsmod
|
||||
|
||||
class _StubMMVUDataset:
|
||||
# MMVU is multimodal in content but, like the real class, does not set
|
||||
# IS_MULTIMODAL -- so it is not subject to the backend gate.
|
||||
IS_MULTIMODAL = False
|
||||
SUPPORTED_DATASET_PATHS = {"yale-nlp/MMVU": lambda x: x["question"]}
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
self.__dict__.update(kwargs)
|
||||
|
||||
def sample(self, *, num_requests, tokenizer, **kwargs): # noqa: ARG002
|
||||
return [
|
||||
SampleRequest(prompt="q", prompt_len=1, expected_output_len=1)
|
||||
for _ in range(num_requests)
|
||||
]
|
||||
|
||||
monkeypatch.setattr(dsmod, "MMVUDataset", _StubMMVUDataset)
|
||||
|
||||
parser = FlexibleArgumentParser()
|
||||
add_cli_args(parser)
|
||||
args = parser.parse_args(
|
||||
[
|
||||
"--dataset-name",
|
||||
"hf",
|
||||
"--dataset-path",
|
||||
"yale-nlp/MMVU",
|
||||
"--backend",
|
||||
"vllm-chat",
|
||||
"--num-prompts",
|
||||
"3",
|
||||
]
|
||||
)
|
||||
requests = get_requests(args, hf_tokenizer)
|
||||
assert len(requests) == 3
|
||||
assert all(isinstance(r, SampleRequest) for r in requests)
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_get_requests_allows_multimodal_on_plain_vllm_backend(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
hf_tokenizer: PreTrainedTokenizerBase,
|
||||
) -> None:
|
||||
"""Regression: --backend vllm must accept multimodal datasets such as ASR.
|
||||
|
||||
_run_vllm_requests forwards multi_modal_data straight to LLM.generate, so
|
||||
the vllm-chat-only gate in get_samples rejected a backend that always
|
||||
worked.
|
||||
"""
|
||||
import vllm.benchmarks.datasets.datasets as dsmod
|
||||
|
||||
class _StubASRDataset:
|
||||
IS_MULTIMODAL = True
|
||||
SUPPORTED_DATASET_PATHS = {"openslr/librispeech_asr"}
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
self.__dict__.update(kwargs)
|
||||
|
||||
def sample(self, *, num_requests, tokenizer, **kwargs): # noqa: ARG002
|
||||
return [
|
||||
SampleRequest(prompt="q", prompt_len=1, expected_output_len=1)
|
||||
for _ in range(num_requests)
|
||||
]
|
||||
|
||||
monkeypatch.setattr(dsmod, "ASRDataset", _StubASRDataset)
|
||||
|
||||
parser = FlexibleArgumentParser()
|
||||
add_cli_args(parser)
|
||||
args = parser.parse_args(
|
||||
[
|
||||
"--dataset-name",
|
||||
"hf",
|
||||
"--dataset-path",
|
||||
"openslr/librispeech_asr",
|
||||
"--backend",
|
||||
"vllm",
|
||||
"--num-prompts",
|
||||
"3",
|
||||
]
|
||||
)
|
||||
requests = get_requests(args, hf_tokenizer)
|
||||
assert len(requests) == 3
|
||||
assert all(isinstance(r, SampleRequest) for r in requests)
|
||||
|
||||
|
||||
def test_to_serve_args_supports_mm_processor_flags() -> None:
|
||||
"""The adapter must serve callers without throughput's legacy flags.
|
||||
|
||||
``bench mm-processor`` exposes neither --input-len/--output-len/--prefix-len
|
||||
nor --backend, and leaves --random-prefix-len at its default of 0. The
|
||||
adapter must not read the legacy flags unconditionally, and must keep the
|
||||
explicit 0 instead of coercing it to None (which later crashed
|
||||
RandomMultiModalDataset.get_prefix on ``None <= 0``).
|
||||
"""
|
||||
args = Namespace(
|
||||
random_input_len=32,
|
||||
random_output_len=1,
|
||||
random_prefix_len=0,
|
||||
)
|
||||
serve_args = _to_serve_args(args)
|
||||
|
||||
assert serve_args.random_input_len == 32
|
||||
assert serve_args.random_output_len == 1
|
||||
assert serve_args.random_prefix_len == 0
|
||||
assert serve_args.backend == "vllm-chat"
|
||||
assert serve_args.enable_multimodal_chat
|
||||
assert serve_args.no_oversample is False
|
||||
|
||||
|
||||
def test_to_serve_args_keeps_legacy_flag_fallback() -> None:
|
||||
"""Callers that only set the legacy flags still get them mapped."""
|
||||
args = Namespace(
|
||||
input_len=256,
|
||||
output_len=8,
|
||||
prefix_len=16,
|
||||
backend="vllm",
|
||||
)
|
||||
serve_args = _to_serve_args(args)
|
||||
|
||||
assert serve_args.random_input_len == 256
|
||||
assert serve_args.random_output_len == 8
|
||||
assert serve_args.random_prefix_len == 16
|
||||
assert serve_args.backend == "vllm"
|
||||
assert not serve_args.enable_multimodal_chat
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_get_requests_random_mm_with_mm_processor_args(
|
||||
hf_tokenizer: PreTrainedTokenizerBase,
|
||||
) -> None:
|
||||
"""Regression: bench mm-processor's random-mm sampling must not crash.
|
||||
|
||||
get_requests feeds mm-processor's parsed args (no legacy length/backend
|
||||
flags) through the shared get_samples dispatch. With the default
|
||||
--random-prefix-len 0 this previously crashed inside
|
||||
RandomMultiModalDataset.get_prefix with a TypeError.
|
||||
"""
|
||||
from vllm.benchmarks.mm_processor import add_cli_args as add_mm_cli_args
|
||||
|
||||
parser = FlexibleArgumentParser()
|
||||
add_mm_cli_args(parser)
|
||||
args = parser.parse_args(
|
||||
[
|
||||
"--dataset-name",
|
||||
"random-mm",
|
||||
"--num-prompts",
|
||||
"3",
|
||||
"--random-input-len",
|
||||
"32",
|
||||
"--random-output-len",
|
||||
"1",
|
||||
"--random-mm-base-items-per-request",
|
||||
"1",
|
||||
"--random-mm-num-mm-items-range-ratio",
|
||||
"0",
|
||||
"--random-mm-limit-mm-per-prompt",
|
||||
'{"image": 1}',
|
||||
"--random-mm-bucket-config",
|
||||
"{(224, 224, 1): 1.0}",
|
||||
]
|
||||
)
|
||||
|
||||
requests = get_requests(args, hf_tokenizer)
|
||||
assert len(requests) == 3
|
||||
assert all(isinstance(r, SampleRequest) for r in requests)
|
||||
# mm-processor drives llm.chat, so prompts must be chat-formatted with
|
||||
# multimodal content attached.
|
||||
for req in requests:
|
||||
assert isinstance(req.prompt, list)
|
||||
assert req.prompt[0]["role"] == "user"
|
||||
assert any(item.get("type") == "image_url" for item in req.prompt[0]["content"])
|
||||
@@ -1,68 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from transformers import AutoTokenizer, PreTrainedTokenizerBase
|
||||
|
||||
from vllm.benchmarks.datasets import CustomDataset
|
||||
from vllm.benchmarks.datasets.create_txt_slices_dataset import create_txt_slices_jsonl
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def hf_tokenizer() -> PreTrainedTokenizerBase:
|
||||
# Use a small, commonly available tokenizer
|
||||
return AutoTokenizer.from_pretrained("openai-community/gpt2")
|
||||
|
||||
|
||||
text_content = """
|
||||
Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor
|
||||
incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud
|
||||
exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.
|
||||
Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat
|
||||
nulla pariatur. Excepteur sint occaecat cupidatat non proident,
|
||||
sunt in culpa qui officia deserunt mollit anim id est laborum.
|
||||
"""
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_create_txt_slices_jsonl(
|
||||
hf_tokenizer: PreTrainedTokenizerBase, tmp_path: Path
|
||||
) -> None:
|
||||
"""Test that create_txt_slices_jsonl produces valid JSONL for CustomDataset."""
|
||||
txt_path = tmp_path / "input.txt"
|
||||
jsonl_path = tmp_path / "input.txt.jsonl"
|
||||
|
||||
txt_path.write_text(text_content)
|
||||
|
||||
create_txt_slices_jsonl(
|
||||
input_path=str(txt_path),
|
||||
output_path=str(jsonl_path),
|
||||
tokenizer_name="openai-community/gpt2",
|
||||
num_prompts=10,
|
||||
input_len=10,
|
||||
output_len=10,
|
||||
)
|
||||
|
||||
# Verify the JSONL file is valid and has the expected structure
|
||||
records = [json.loads(line) for line in jsonl_path.read_text().splitlines()]
|
||||
|
||||
assert len(records) == 10
|
||||
for record in records:
|
||||
assert "prompt" in record
|
||||
assert "output_tokens" in record
|
||||
assert isinstance(record["prompt"], str)
|
||||
assert record["output_tokens"] == 10
|
||||
|
||||
# Verify the JSONL file can be loaded by CustomDataset
|
||||
dataset = CustomDataset(dataset_path=str(jsonl_path))
|
||||
samples = dataset.sample(
|
||||
tokenizer=hf_tokenizer,
|
||||
num_requests=10,
|
||||
output_len=10,
|
||||
skip_chat_template=True,
|
||||
)
|
||||
|
||||
assert len(samples) == 10
|
||||
assert all(sample.expected_output_len == 10 for sample in samples)
|
||||
@@ -1,71 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import (
|
||||
CompilationConfig,
|
||||
CompilationMode,
|
||||
VllmConfig,
|
||||
set_current_vllm_config,
|
||||
)
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.model_executor.models.qwen3_vl import (
|
||||
Qwen3VLForConditionalGeneration,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
|
||||
@support_torch_compile(
|
||||
dynamic_arg_dims={"x": 0, "deepstack_input_embeds": 0},
|
||||
)
|
||||
class _DeepStackDecoder(nn.Module):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
super().__init__()
|
||||
|
||||
def forward(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
deepstack_input_embeds: IntermediateTensors | None,
|
||||
) -> torch.Tensor:
|
||||
if deepstack_input_embeds is not None:
|
||||
x = x + deepstack_input_embeds["deepstack_input_embeds_0"]
|
||||
return x
|
||||
|
||||
|
||||
def _run_contract(
|
||||
deepstack_buffer: Qwen3VLForConditionalGeneration,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
config = VllmConfig(
|
||||
compilation_config=CompilationConfig(mode=CompilationMode.VLLM_COMPILE)
|
||||
)
|
||||
with set_current_vllm_config(config):
|
||||
model = _DeepStackDecoder(vllm_config=config).eval().cuda()
|
||||
|
||||
x = torch.arange(8, dtype=torch.float32, device="cuda").reshape(2, 4)
|
||||
with torch.inference_mode(), set_forward_context(None, vllm_config=config):
|
||||
empty_payload = deepstack_buffer._get_deepstack_input_embeds(num_tokens=2)
|
||||
first = model(x, empty_payload)
|
||||
|
||||
payload = torch.full((1, 2, 4), 7.0, device="cuda")
|
||||
deepstack_buffer._set_deepstack_input_embeds(payload)
|
||||
active_payload = deepstack_buffer._get_deepstack_input_embeds(num_tokens=2)
|
||||
assert active_payload is not None
|
||||
second = model(x, active_payload)
|
||||
|
||||
return x.cpu(), first.cpu(), second.cpu()
|
||||
|
||||
|
||||
def test_qwen3_vl_deepstack_payload_after_empty_compile(monkeypatch):
|
||||
monkeypatch.setenv("VLLM_DISABLE_COMPILE_CACHE", "1")
|
||||
|
||||
deepstack_buffer = object.__new__(Qwen3VLForConditionalGeneration)
|
||||
deepstack_buffer.deepstack_input_embeds = [torch.zeros(2, 4, device="cuda")]
|
||||
deepstack_buffer.deepstack_input_embeds_num_tokens = 0
|
||||
deepstack_buffer.deepstack_num_level = 1
|
||||
|
||||
x, first, second = _run_contract(deepstack_buffer)
|
||||
torch.testing.assert_close(first, x)
|
||||
torch.testing.assert_close(second, x + 7)
|
||||
@@ -1,727 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import random
|
||||
|
||||
import pytest
|
||||
import ray
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
|
||||
from vllm.distributed.communication_op import tensor_model_parallel_all_reduce # noqa
|
||||
from vllm.distributed.device_communicators import custom_all_reduce as car
|
||||
from vllm.distributed.parallel_state import get_tp_group, graph_capture
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ..utils import (
|
||||
ensure_model_parallel_initialized,
|
||||
init_test_distributed_environment,
|
||||
multi_process_parallel,
|
||||
)
|
||||
|
||||
random.seed(42)
|
||||
test_sizes = [random.randint(1024, 2048 * 1024) for _ in range(8)]
|
||||
for i, v in enumerate(test_sizes):
|
||||
test_sizes[i] -= v % 8
|
||||
|
||||
|
||||
def _bf16_ulps(a: torch.Tensor, b: torch.Tensor) -> int:
|
||||
"""Largest distance between two BF16 tensors in units in the last place."""
|
||||
|
||||
def ordered(t: torch.Tensor) -> torch.Tensor:
|
||||
bits = t.view(torch.int16).int() & 0xFFFF
|
||||
return torch.where(bits >= 0x8000, 0x8000 - bits, bits)
|
||||
|
||||
return int((ordered(a) - ordered(b)).abs().max())
|
||||
|
||||
|
||||
def _unfused_all_reduce_mhc(peers, residual, post, comb, pre, weight, eps):
|
||||
"""All-reduce, mHC post, collapse and RMSNorm in the fused kernel's FP32
|
||||
order, rounding to BF16 where the unfused path returns BF16."""
|
||||
reduced = peers[0].float()
|
||||
for peer in peers[1:]:
|
||||
reduced = reduced + peer.float()
|
||||
reduced = reduced.bfloat16().float()
|
||||
output = torch.empty_like(residual)
|
||||
collapse = torch.zeros_like(reduced)
|
||||
for target in range(4):
|
||||
mixed = reduced * post[:, target : target + 1]
|
||||
for source in range(4):
|
||||
mixed = torch.addcmul(
|
||||
mixed, residual[:, source].float(), comb[:, source, target, None]
|
||||
)
|
||||
output[:, target] = mixed.bfloat16()
|
||||
collapse = torch.addcmul(
|
||||
collapse, output[:, target].float(), pre[:, target : target + 1]
|
||||
)
|
||||
prenorm = collapse.bfloat16().float()
|
||||
inv_rms = torch.rsqrt(prenorm.square().mean(-1, keepdim=True) + eps)
|
||||
return output, (prenorm * inv_rms * weight.float()).bfloat16()
|
||||
|
||||
|
||||
@ray.remote(num_gpus=1, max_calls=1)
|
||||
def _all_reduce_mhc(monkeypatch, tp_size, pp_size, rank, distributed_init_port):
|
||||
from vllm.models.deepseek_v41.nvidia.ops.cute_dsl import AllReduceMHC
|
||||
|
||||
with monkeypatch.context() as m:
|
||||
m.delenv("CUDA_VISIBLE_DEVICES", raising=False)
|
||||
device = torch.device(f"cuda:{rank}")
|
||||
torch.accelerator.set_device_index(device)
|
||||
init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port)
|
||||
ensure_model_parallel_initialized(tp_size, pp_size)
|
||||
op = AllReduceMHC(
|
||||
hidden_size=5120, hc_mult=4, max_num_tokens=16, top_k=6, device=device
|
||||
)
|
||||
|
||||
def check_eager_and_replayed(fused, check, halved, doubled):
|
||||
check(fused())
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
for _ in range(5):
|
||||
fused()
|
||||
captured = fused()
|
||||
for _ in range(20):
|
||||
graph.replay()
|
||||
check(captured)
|
||||
# Replays must pick up in-place input changes.
|
||||
halved.mul_(0.5)
|
||||
doubled.mul_(2)
|
||||
graph.replay()
|
||||
check(captured)
|
||||
|
||||
def mhc_inputs(n):
|
||||
torch.manual_seed(123)
|
||||
residual = torch.randn(n, 4, 5120, device=device, dtype=torch.bfloat16)
|
||||
post = torch.rand(n, 4, device=device)
|
||||
comb = torch.randn(n, 4, 4, device=device) * 0.1
|
||||
pre = torch.rand(n, 4, device=device)
|
||||
weight = torch.randn(5120, device=device, dtype=torch.bfloat16)
|
||||
return residual, post, comb, pre, weight
|
||||
|
||||
def run(n):
|
||||
torch.manual_seed(42 + rank)
|
||||
x = torch.randn(n, 5120, device=device, dtype=torch.bfloat16)
|
||||
# Packed +0/-0 pairs collide with the Lamport sentinel.
|
||||
x[:, :16] = 0
|
||||
x[:, 9:16:2] = -0.0
|
||||
residual, post, comb, pre, weight = mhc_inputs(n)
|
||||
|
||||
def fused():
|
||||
return op(x, residual, post, comb, pre, weight, 1e-6)
|
||||
|
||||
def check(outputs):
|
||||
peers = get_tp_group().all_gather(x, dim=0).view(tp_size, n, 5120)
|
||||
output, normalized = _unfused_all_reduce_mhc(
|
||||
peers, residual, post, comb, pre, weight, 1e-6
|
||||
)
|
||||
# The mixed hc streams match bit for bit. The RMSNorm sums the
|
||||
# squares in another order and uses an approximate rsqrt.
|
||||
assert torch.equal(outputs[0], output)
|
||||
assert _bf16_ulps(outputs[1], normalized) <= 1
|
||||
|
||||
check_eager_and_replayed(fused, check, x, residual)
|
||||
|
||||
def run_finalize(n):
|
||||
torch.manual_seed(7 + rank)
|
||||
# A padded permuted GEMM2 buffer, like the MoE's.
|
||||
rows = n * 6 + 5
|
||||
gemm2 = torch.randn(rows, 5120, device=device, dtype=torch.bfloat16)
|
||||
permuted = torch.randperm(rows, device=device)[: n * 6].view(n, 6).int()
|
||||
# A route to an expert this rank does not hold.
|
||||
permuted[0, -1] = -1
|
||||
weights = torch.rand(n, 6, device=device)
|
||||
shared = torch.randn(n, 5120, device=device, dtype=torch.bfloat16)
|
||||
residual, post, comb, pre, weight = mhc_inputs(n)
|
||||
|
||||
def fused():
|
||||
return op.finalize(
|
||||
gemm2,
|
||||
weights,
|
||||
permuted,
|
||||
shared,
|
||||
residual,
|
||||
post,
|
||||
comb,
|
||||
pre,
|
||||
weight,
|
||||
1e-6,
|
||||
)
|
||||
|
||||
def check(outputs):
|
||||
# One FP32 FMA per route in route order, the shared add, one
|
||||
# rounding, then the plain path: only the finalize differs.
|
||||
acc = torch.zeros(n, 5120, device=device)
|
||||
for k in range(6):
|
||||
valid = (permuted[:, k] >= 0).unsqueeze(-1)
|
||||
rows_k = gemm2[permuted[:, k].clamp_min(0).long()].float()
|
||||
torch.addcmul(
|
||||
acc,
|
||||
torch.where(valid, rows_k, 0.0),
|
||||
torch.where(valid, weights[:, k : k + 1], 0.0),
|
||||
out=acc,
|
||||
)
|
||||
x = (acc + shared.float()).bfloat16()
|
||||
expected = op(x, residual, post, comb, pre, weight, 1e-6)
|
||||
assert torch.equal(outputs[0], expected[0])
|
||||
assert torch.equal(outputs[1], expected[1])
|
||||
|
||||
check_eager_and_replayed(fused, check, gemm2, shared)
|
||||
|
||||
# Changing shapes cover shrinking and growing batches in one mailbox.
|
||||
for n in (1, 6, 12, 8, 16, 3, 5, 2, 4, 1):
|
||||
run(n)
|
||||
run_finalize(n)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_device_capability_family(100), reason="Requires SM100"
|
||||
)
|
||||
def test_all_reduce_mhc_matches_unfused_path(monkeypatch):
|
||||
if torch.accelerator.device_count() < 4:
|
||||
pytest.skip("Requires four GPUs with NVLink multicast")
|
||||
multi_process_parallel(monkeypatch, 4, 1, _all_reduce_mhc)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("dtype", "expected"),
|
||||
[
|
||||
(torch.float32, True),
|
||||
(torch.float16, True),
|
||||
(torch.bfloat16, True),
|
||||
(torch.int8, False),
|
||||
(torch.float8_e4m3fn, False),
|
||||
],
|
||||
)
|
||||
def test_custom_allreduce_filters_dtype(
|
||||
dtype: torch.dtype,
|
||||
expected: bool,
|
||||
) -> None:
|
||||
communicator = car.CustomAllreduce.__new__(car.CustomAllreduce)
|
||||
communicator.disabled = False
|
||||
communicator._ptr = 0
|
||||
communicator.world_size = 2
|
||||
communicator.max_size = 1024
|
||||
|
||||
assert communicator.should_custom_ar(torch.empty(16, dtype=dtype)) is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("batch_invariant", [False, True])
|
||||
def test_custom_allreduce_size_gate_ignored_under_batch_invariance(
|
||||
batch_invariant: bool,
|
||||
) -> None:
|
||||
"""Batch invariance must not switch backends based on the input size."""
|
||||
communicator = car.CustomAllreduce.__new__(car.CustomAllreduce)
|
||||
communicator.disabled = False
|
||||
communicator.world_size = 2
|
||||
communicator.max_size = 1024
|
||||
communicator.batch_invariant = batch_invariant
|
||||
communicator._ptr = 0
|
||||
|
||||
oversized = torch.empty(1024, dtype=torch.float16)
|
||||
assert communicator.should_custom_ar(oversized) is batch_invariant
|
||||
|
||||
|
||||
@pytest.mark.parametrize("batch_invariant", [False, True])
|
||||
def test_custom_reduce_scatter_disabled_under_batch_invariance(
|
||||
monkeypatch, batch_invariant: bool
|
||||
) -> None:
|
||||
"""Reduce-scatter stays on one backend under batch invariance."""
|
||||
monkeypatch.setattr(car.current_platform, "is_cuda", lambda: True)
|
||||
communicator = car.CustomAllreduce.__new__(car.CustomAllreduce)
|
||||
communicator.disabled = False
|
||||
communicator.world_size = 2
|
||||
communicator.fully_connected = True
|
||||
communicator.mnnvl_only = False
|
||||
communicator.mnnvl_multicast_ptr = 0
|
||||
communicator.max_reduce_scatter_size = 1024
|
||||
communicator.max_mnnvl_reduce_scatter_size = 1024
|
||||
communicator.batch_invariant = batch_invariant
|
||||
|
||||
in_range = torch.empty(64, dtype=torch.float16)
|
||||
assert communicator.should_custom_reduce_scatter(in_range) is not batch_invariant
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("major", "local_multicast", "expected"),
|
||||
[
|
||||
(8, True, False),
|
||||
(9, True, False),
|
||||
(10, False, False),
|
||||
(10, True, True),
|
||||
],
|
||||
)
|
||||
def test_cross_node_mnnvl_gate_checks_generation_and_multicast(
|
||||
monkeypatch,
|
||||
major,
|
||||
local_multicast,
|
||||
expected,
|
||||
):
|
||||
def has_device_capability(capability, device_id):
|
||||
assert capability == 100
|
||||
assert device_id == 3
|
||||
return major >= 10
|
||||
|
||||
monkeypatch.setattr(
|
||||
car.current_platform,
|
||||
"has_device_capability",
|
||||
has_device_capability,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
car,
|
||||
"_has_local_multicast_support",
|
||||
lambda _device: local_multicast,
|
||||
)
|
||||
monkeypatch.setattr(car.dist, "all_reduce", lambda *_args, **_kwargs: None)
|
||||
|
||||
assert car._group_can_attempt_mnnvl(object(), torch.device("cuda:3")) is expected
|
||||
|
||||
|
||||
def test_cross_node_mnnvl_gate_requires_support_on_every_rank(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
car.current_platform,
|
||||
"has_device_capability",
|
||||
lambda *_args: True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
car,
|
||||
"_has_local_multicast_support",
|
||||
lambda _device: True,
|
||||
)
|
||||
|
||||
def report_unsupported_peer(support, **_kwargs):
|
||||
support.zero_()
|
||||
|
||||
monkeypatch.setattr(car.dist, "all_reduce", report_unsupported_peer)
|
||||
|
||||
assert not car._group_can_attempt_mnnvl(object(), torch.device("cuda:0"))
|
||||
|
||||
|
||||
def test_local_multicast_support_rejects_non_cuda(monkeypatch):
|
||||
monkeypatch.setattr(car.current_platform, "is_cuda", lambda: False)
|
||||
|
||||
assert not car._has_local_multicast_support(torch.device("cuda:0"))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("world_size", "device_capability", "local_multicast", "expected"),
|
||||
[
|
||||
(2, (10, 0), True, True),
|
||||
(4, (10, 3), True, True),
|
||||
(8, (10, 0), True, True),
|
||||
(8, (10, 3), True, True),
|
||||
(6, (10, 3), True, False),
|
||||
(8, (10, 1), True, False),
|
||||
(8, (9, 0), True, False),
|
||||
(8, (10, 3), False, False),
|
||||
],
|
||||
)
|
||||
def test_mnnvl_multimem_reduce_scatter_platform_gate(
|
||||
monkeypatch,
|
||||
world_size,
|
||||
device_capability,
|
||||
local_multicast,
|
||||
expected,
|
||||
):
|
||||
def is_device_capability(capability, device_id):
|
||||
assert capability in ((10, 0), (10, 3))
|
||||
assert device_id == 3
|
||||
return device_capability == capability
|
||||
|
||||
monkeypatch.setattr(
|
||||
car.current_platform,
|
||||
"is_device_capability",
|
||||
is_device_capability,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
car,
|
||||
"_has_local_multicast_support",
|
||||
lambda _device: local_multicast,
|
||||
)
|
||||
|
||||
supported = car._supports_mnnvl_multimem_reduce_scatter(
|
||||
torch.device("cuda:3"), world_size
|
||||
)
|
||||
assert supported is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
(
|
||||
"message_bytes",
|
||||
"multimem_ptr",
|
||||
"multimem_initialized",
|
||||
"batch_invariant",
|
||||
"expected",
|
||||
),
|
||||
[
|
||||
(16 * 1024 * 1024, 1, True, False, "mnnvl_lamport"),
|
||||
(16 * 1024 * 1024 + 128, 1, True, False, "mnnvl_multimem"),
|
||||
(64 * 1024 * 1024, 1, True, False, "mnnvl_multimem"),
|
||||
(64 * 1024 * 1024 + 128, 1, True, False, None),
|
||||
(32 * 1024 * 1024, 0, True, False, None),
|
||||
(32 * 1024 * 1024, 0, False, False, "mnnvl_multimem"),
|
||||
(8 * 1024 * 1024, 1, True, True, "mnnvl_lamport"),
|
||||
(32 * 1024 * 1024, 1, True, True, None),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("world_size", [2, 4, 8])
|
||||
def test_mnnvl_reduce_scatter_backend_gate(
|
||||
monkeypatch,
|
||||
world_size,
|
||||
message_bytes,
|
||||
multimem_ptr,
|
||||
multimem_initialized,
|
||||
batch_invariant,
|
||||
expected,
|
||||
):
|
||||
monkeypatch.setattr(car.current_platform, "is_cuda", lambda: True)
|
||||
monkeypatch.setattr(car.envs, "VLLM_BATCH_INVARIANT", batch_invariant)
|
||||
communicator = car.CustomAllreduce.__new__(car.CustomAllreduce)
|
||||
communicator.disabled = False
|
||||
communicator._ptr = 0
|
||||
communicator.world_size = world_size
|
||||
communicator.mnnvl_only = False
|
||||
communicator.fully_connected = True
|
||||
communicator.mnnvl_multicast_ptr = 1
|
||||
communicator.mnnvl_multimem_rs_supported = True
|
||||
communicator.mnnvl_multimem_rs_initialized = multimem_initialized
|
||||
communicator.mnnvl_multimem_rs_multicast_ptr = multimem_ptr
|
||||
communicator.max_mnnvl_reduce_scatter_size = 16 * 1024 * 1024
|
||||
communicator.max_mnnvl_multimem_reduce_scatter_size = 64 * 1024 * 1024
|
||||
communicator.max_reduce_scatter_size = 16 * 1024 * 1024
|
||||
inp = torch.empty(
|
||||
(world_size, message_bytes // torch.bfloat16.itemsize // world_size),
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
|
||||
assert inp.nbytes == message_bytes
|
||||
assert communicator._select_reduce_scatter_backend(inp) == expected
|
||||
assert communicator.should_custom_reduce_scatter(inp) is (expected is not None)
|
||||
assert communicator.should_mnnvl_multimem_reduce_scatter(inp) is (
|
||||
expected == "mnnvl_multimem"
|
||||
)
|
||||
|
||||
|
||||
def test_mnnvl_multimem_reduce_scatter_skips_rendezvous_after_peer_alloc_failure(
|
||||
monkeypatch,
|
||||
):
|
||||
events = []
|
||||
|
||||
class FakeSymmMem:
|
||||
@staticmethod
|
||||
def empty(*_args, **_kwargs):
|
||||
events.append("empty")
|
||||
return torch.empty(1, dtype=torch.uint8)
|
||||
|
||||
@staticmethod
|
||||
def rendezvous(*_args, **_kwargs):
|
||||
events.append("rendezvous")
|
||||
return None
|
||||
|
||||
def report_peer_allocation_failure(group_value, **_kwargs):
|
||||
events.append("all_reduce")
|
||||
assert group_value.item() == 1
|
||||
group_value.zero_()
|
||||
|
||||
monkeypatch.setattr(car, "torch_symm_mem", FakeSymmMem)
|
||||
monkeypatch.setattr(car.ops, "meta_size", lambda: 128)
|
||||
monkeypatch.setattr(car.dist, "all_reduce", report_peer_allocation_failure)
|
||||
warnings = []
|
||||
monkeypatch.setattr(
|
||||
car.logger,
|
||||
"warning_once",
|
||||
lambda message, *_args, **_kwargs: warnings.append(message),
|
||||
)
|
||||
|
||||
communicator = car.CustomAllreduce.__new__(car.CustomAllreduce)
|
||||
communicator.disabled = True
|
||||
communicator._ptr = 0
|
||||
communicator.group = object()
|
||||
communicator.device = torch.device("cuda:0")
|
||||
communicator.max_mnnvl_multimem_reduce_scatter_size = 64 * 1024 * 1024
|
||||
communicator.mnnvl_multimem_rs_supported = True
|
||||
communicator.mnnvl_multimem_rs_initialized = False
|
||||
communicator.mnnvl_multimem_rs_buffer = None
|
||||
communicator.mnnvl_multimem_rs_multicast_ptr = 0
|
||||
|
||||
communicator._init_mnnvl_multimem_reduce_scatter_buffer()
|
||||
|
||||
assert events == ["empty", "all_reduce"]
|
||||
assert communicator.mnnvl_multimem_rs_initialized
|
||||
assert communicator.mnnvl_multimem_rs_buffer is None
|
||||
assert communicator.mnnvl_multimem_rs_multicast_ptr == 0
|
||||
assert warnings == [
|
||||
"MNNVL multimem reduce-scatter symmetric-memory allocation "
|
||||
"failed on at least one rank; falling back to NCCL."
|
||||
]
|
||||
|
||||
|
||||
def test_mnnvl_multimem_reduce_scatter_warns_on_rendezvous_failure(monkeypatch):
|
||||
events = []
|
||||
|
||||
class FakeSymmMem:
|
||||
@staticmethod
|
||||
def empty(*_args, **_kwargs):
|
||||
events.append("empty")
|
||||
return torch.empty(1, dtype=torch.uint8)
|
||||
|
||||
@staticmethod
|
||||
def rendezvous(*_args, **_kwargs):
|
||||
events.append("rendezvous")
|
||||
raise RuntimeError("rendezvous failed")
|
||||
|
||||
def preserve_local_result(_group_value, **_kwargs):
|
||||
events.append("all_reduce")
|
||||
|
||||
warnings = []
|
||||
monkeypatch.setattr(car, "torch_symm_mem", FakeSymmMem)
|
||||
monkeypatch.setattr(car.ops, "meta_size", lambda: 128)
|
||||
monkeypatch.setattr(car.dist, "all_reduce", preserve_local_result)
|
||||
monkeypatch.setattr(
|
||||
car.logger,
|
||||
"warning_once",
|
||||
lambda message, *_args, **_kwargs: warnings.append(message),
|
||||
)
|
||||
|
||||
communicator = car.CustomAllreduce.__new__(car.CustomAllreduce)
|
||||
communicator.disabled = True
|
||||
communicator._ptr = 0
|
||||
communicator.group = type("Group", (), {"group_name": "test"})()
|
||||
communicator.device = torch.device("cuda:0")
|
||||
communicator.max_mnnvl_multimem_reduce_scatter_size = 64 * 1024 * 1024
|
||||
communicator.mnnvl_multimem_rs_supported = True
|
||||
communicator.mnnvl_multimem_rs_initialized = False
|
||||
communicator.mnnvl_multimem_rs_buffer = None
|
||||
communicator.mnnvl_multimem_rs_multicast_ptr = 0
|
||||
|
||||
communicator._init_mnnvl_multimem_reduce_scatter_buffer()
|
||||
|
||||
assert events == ["empty", "all_reduce", "rendezvous", "all_reduce"]
|
||||
assert communicator.mnnvl_multimem_rs_initialized
|
||||
assert communicator.mnnvl_multimem_rs_buffer is None
|
||||
assert communicator.mnnvl_multimem_rs_multicast_ptr == 0
|
||||
assert warnings == [
|
||||
"MNNVL multimem reduce-scatter symmetric-memory rendezvous "
|
||||
"failed on at least one rank; falling back to NCCL."
|
||||
]
|
||||
|
||||
|
||||
def test_mnnvl_multimem_reduce_scatter_initializes_signals(monkeypatch):
|
||||
events = []
|
||||
buffers = []
|
||||
|
||||
class FakeHandle:
|
||||
multicast_ptr = 0x3000
|
||||
|
||||
class FakeSymmMem:
|
||||
@staticmethod
|
||||
def empty(size, **_kwargs):
|
||||
events.append(("empty", size))
|
||||
buffer = torch.ones(size, dtype=torch.uint8)
|
||||
buffers.append(buffer)
|
||||
return buffer
|
||||
|
||||
@staticmethod
|
||||
def rendezvous(*_args, **_kwargs):
|
||||
events.append(("rendezvous", None))
|
||||
return FakeHandle()
|
||||
|
||||
def preserve_local_result(*_args, **_kwargs):
|
||||
events.append(("all_reduce", None))
|
||||
|
||||
monkeypatch.setattr(car, "torch_symm_mem", FakeSymmMem)
|
||||
monkeypatch.setattr(car.ops, "meta_size", lambda: 128)
|
||||
monkeypatch.setattr(
|
||||
car.torch.accelerator,
|
||||
"synchronize",
|
||||
lambda: events.append(("synchronize", None)),
|
||||
)
|
||||
monkeypatch.setattr(car.dist, "all_reduce", preserve_local_result)
|
||||
|
||||
communicator = car.CustomAllreduce.__new__(car.CustomAllreduce)
|
||||
communicator.disabled = True
|
||||
communicator._ptr = 0
|
||||
communicator.group = type("Group", (), {"group_name": "test"})()
|
||||
communicator.device = torch.device("cpu")
|
||||
communicator.max_mnnvl_multimem_reduce_scatter_size = 129
|
||||
communicator.mnnvl_multimem_rs_supported = True
|
||||
communicator.mnnvl_multimem_rs_initialized = False
|
||||
communicator.mnnvl_multimem_rs_buffer = None
|
||||
communicator.mnnvl_multimem_rs_multicast_ptr = 0
|
||||
|
||||
communicator._init_mnnvl_multimem_reduce_scatter_buffer()
|
||||
|
||||
assert events == [
|
||||
("empty", 257),
|
||||
("all_reduce", None),
|
||||
("rendezvous", None),
|
||||
("synchronize", None),
|
||||
("all_reduce", None),
|
||||
]
|
||||
assert torch.all(buffers[0][:128] == 0)
|
||||
assert torch.all(buffers[0][128:] == 1)
|
||||
assert communicator.mnnvl_multimem_rs_buffer_size == 129
|
||||
assert communicator.mnnvl_multimem_rs_local_ptr == buffers[0].data_ptr() + 128
|
||||
assert communicator.mnnvl_multimem_rs_multicast_ptr == 0x3080
|
||||
|
||||
|
||||
@ray.remote(num_gpus=1, max_calls=1)
|
||||
def graph_allreduce(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tp_size,
|
||||
pp_size,
|
||||
rank,
|
||||
distributed_init_port,
|
||||
):
|
||||
with monkeypatch.context() as m:
|
||||
m.delenv("CUDA_VISIBLE_DEVICES", raising=False)
|
||||
m.delenv("HIP_VISIBLE_DEVICES", raising=False)
|
||||
device = torch.device(f"cuda:{rank}")
|
||||
torch.accelerator.set_device_index(device)
|
||||
init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port)
|
||||
ensure_model_parallel_initialized(tp_size, pp_size)
|
||||
group = get_tp_group().device_group
|
||||
|
||||
# A small all_reduce for warmup.
|
||||
# this is needed because device communicators might be created lazily
|
||||
# (e.g. NCCL). This will ensure that the communicator is initialized
|
||||
# before any communication happens, so that this group can be used for
|
||||
# graph capture immediately.
|
||||
data = torch.zeros(1)
|
||||
data = data.to(device=device)
|
||||
torch.distributed.all_reduce(data, group=group)
|
||||
torch.accelerator.synchronize()
|
||||
del data
|
||||
|
||||
# we use the first group to communicate once
|
||||
# and the second group to communicate twice
|
||||
# and so on
|
||||
# this is used to demonstrate that each group can
|
||||
# communicate independently
|
||||
num_communication = rank // tp_size + 1
|
||||
|
||||
for sz in test_sizes:
|
||||
for dtype in [torch.float32, torch.float16, torch.bfloat16]:
|
||||
with graph_capture(device=device) as graph_capture_context:
|
||||
# use integers so result matches NCCL exactly
|
||||
device_idx = torch.accelerator.current_device_index()
|
||||
inp1 = torch.randint(1, 16, (sz,), dtype=dtype, device=device_idx)
|
||||
inp2 = torch.randint(1, 16, (sz,), dtype=dtype, device=device_idx)
|
||||
|
||||
torch.accelerator.synchronize()
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph, stream=graph_capture_context.stream):
|
||||
for i in range(num_communication):
|
||||
out1 = tensor_model_parallel_all_reduce(inp1)
|
||||
# the input buffer is immediately modified to test
|
||||
# synchronization
|
||||
dist.all_reduce(inp1, group=group)
|
||||
out2 = tensor_model_parallel_all_reduce(inp2)
|
||||
dist.all_reduce(inp2, group=group)
|
||||
graph.replay()
|
||||
torch.testing.assert_close(out1, inp1)
|
||||
torch.testing.assert_close(out2, inp2)
|
||||
|
||||
|
||||
@ray.remote(num_gpus=1, max_calls=1)
|
||||
def eager_allreduce(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tp_size,
|
||||
pp_size,
|
||||
rank,
|
||||
distributed_init_port,
|
||||
):
|
||||
with monkeypatch.context() as m:
|
||||
m.delenv("CUDA_VISIBLE_DEVICES", raising=False)
|
||||
m.delenv("HIP_VISIBLE_DEVICES", raising=False)
|
||||
device = torch.device(f"cuda:{rank}")
|
||||
torch.accelerator.set_device_index(device)
|
||||
init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port)
|
||||
|
||||
# we use the first group to communicate once
|
||||
# and the second group to communicate twice
|
||||
# and so on
|
||||
# this is used to demonstrate that each group can
|
||||
# communicate independently
|
||||
num_communication = rank // tp_size + 1
|
||||
sz = 1024
|
||||
fa = get_tp_group().device_communicator.ca_comm
|
||||
inp = torch.ones(sz, dtype=torch.float32, device=device)
|
||||
out = inp
|
||||
for _ in range(num_communication):
|
||||
out = fa.all_reduce(out, registered=False)
|
||||
torch.testing.assert_close(out, inp * (tp_size**num_communication))
|
||||
|
||||
inp = torch.ones(sz * 4, dtype=torch.bfloat16, device=device)
|
||||
out = inp
|
||||
for _ in range(num_communication):
|
||||
out = fa.all_reduce(out, registered=False)
|
||||
torch.testing.assert_close(out, inp * (tp_size**num_communication))
|
||||
|
||||
|
||||
@ray.remote(num_gpus=1, max_calls=1)
|
||||
def chunked_allreduce(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tp_size,
|
||||
pp_size,
|
||||
rank,
|
||||
distributed_init_port,
|
||||
):
|
||||
"""Inputs above max_size are reduced in chunks and match the fixed-order
|
||||
reference bitwise, both eagerly and inside a CUDA graph."""
|
||||
with monkeypatch.context() as m:
|
||||
m.delenv("CUDA_VISIBLE_DEVICES", raising=False)
|
||||
m.delenv("HIP_VISIBLE_DEVICES", raising=False)
|
||||
m.setenv("VLLM_CUSTOM_ALLREDUCE_ALGO", "1stage")
|
||||
device = torch.device(f"cuda:{rank}")
|
||||
torch.accelerator.set_device_index(device)
|
||||
init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port)
|
||||
ensure_model_parallel_initialized(tp_size, pp_size)
|
||||
group = get_tp_group().device_group
|
||||
fa = get_tp_group().device_communicator.ca_comm
|
||||
|
||||
# Two full chunks plus a partial one.
|
||||
dtype = torch.bfloat16
|
||||
chunk_numel = fa.max_size // dtype.itemsize
|
||||
inp = torch.randn(2 * chunk_numel + 4096, device=device).to(dtype)
|
||||
gathered = [torch.empty_like(inp) for _ in range(tp_size)]
|
||||
dist.all_gather(gathered, inp, group=group)
|
||||
ref = gathered[0].float()
|
||||
for peer in gathered[1:]:
|
||||
ref = ref + peer.float()
|
||||
ref = ref.to(torch.bfloat16)
|
||||
|
||||
out = fa.all_reduce(inp, registered=False)
|
||||
assert torch.equal(out, ref)
|
||||
|
||||
# Weak-contiguous but not C-contiguous: a transposed matrix.
|
||||
inp_t, ref_t = inp.view(-1, 4096).t(), ref.view(-1, 4096).t()
|
||||
assert torch.equal(fa.all_reduce(inp_t, registered=False), ref_t)
|
||||
|
||||
with fa.capture():
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
out = fa.all_reduce(inp, registered=True)
|
||||
graph.replay()
|
||||
assert torch.equal(out, ref)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tp_size", [2, 4])
|
||||
def test_custom_allreduce_chunked(monkeypatch: pytest.MonkeyPatch, tp_size):
|
||||
if tp_size > torch.accelerator.device_count():
|
||||
pytest.skip("Not enough GPUs to run the test.")
|
||||
multi_process_parallel(monkeypatch, tp_size, 1, chunked_allreduce)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tp_size", [2])
|
||||
@pytest.mark.parametrize("pipeline_parallel_size", [1, 2])
|
||||
@pytest.mark.parametrize("test_target", [eager_allreduce, graph_allreduce])
|
||||
def test_custom_allreduce(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tp_size,
|
||||
pipeline_parallel_size,
|
||||
test_target,
|
||||
):
|
||||
world_size = tp_size * pipeline_parallel_size
|
||||
if world_size > torch.accelerator.device_count():
|
||||
pytest.skip("Not enough GPUs to run the test.")
|
||||
multi_process_parallel(monkeypatch, tp_size, pipeline_parallel_size, test_target)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,883 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Engram tables shared by co-located DP replicas.
|
||||
|
||||
Four GPUs on one node split the hash heads four ways, whether as DEP
|
||||
(TP1 x DP4, the single-node MoE layout) or TP2 x DP2. A lookup only returns
|
||||
the right rows if the ids of every replica are gathered, each rank looks up
|
||||
its own heads, and each replica trades those tokens back for the heads its
|
||||
peers own.
|
||||
|
||||
With shared host storage, DP replicas instead map the same TP slice and
|
||||
prefetch only their own tokens without DP gathers.
|
||||
"""
|
||||
|
||||
from itertools import product
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
from threading import Event
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.multiprocessing as mp
|
||||
|
||||
from vllm.config import (
|
||||
EngramConfig,
|
||||
VllmConfig,
|
||||
get_current_vllm_config_or_none,
|
||||
set_current_vllm_config,
|
||||
)
|
||||
from vllm.config.load import LoadConfig
|
||||
from vllm.config.parallel import ParallelConfig
|
||||
from vllm.config.scheduler import SchedulerConfig
|
||||
from vllm.distributed import cleanup_dist_env_and_memory, parallel_state
|
||||
from vllm.distributed.parallel_state import (
|
||||
get_engram_dp_group,
|
||||
get_engram_dp_size,
|
||||
init_distributed_environment,
|
||||
initialize_model_parallel,
|
||||
)
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
||||
dequant_mxfp8_to_bf16,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import (
|
||||
convert_swizzled_to_linear,
|
||||
)
|
||||
from vllm.models.deepseek_v41.common import engram as engram_ops
|
||||
from vllm.models.deepseek_v41.common.engram import (
|
||||
Engram,
|
||||
EngramBatch,
|
||||
EngramLayout,
|
||||
ParallelEngramEmbedding,
|
||||
engram_head_shard_rank,
|
||||
gather_engram_hashes,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
from vllm.utils.system_utils import update_environment_variables
|
||||
|
||||
WORLD_SIZE = 4
|
||||
DIM, BLOCK = 64, 32
|
||||
# Prime-sized buckets, as the model's layout builds them.
|
||||
HEAD_SIZES = (97, 101, 103, 107, 109, 113, 127, 131)
|
||||
|
||||
|
||||
def _token_counts(dp_size: int) -> tuple[tuple[int, ...], ...]:
|
||||
"""Even loads, then uneven ones with an idle replica to pad around."""
|
||||
return ((8,) * dp_size, (7, 0, 5, 1)[:dp_size], (7,) * dp_size)
|
||||
|
||||
|
||||
def _full_table(rows: int) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""The unsharded table, identical on every rank."""
|
||||
torch.manual_seed(0)
|
||||
weight = (torch.randn(rows, DIM) * 4).to(torch.float8_e4m3fn)
|
||||
scale_inv = torch.randint(120, 134, (rows, DIM // BLOCK), dtype=torch.uint8)
|
||||
return weight, scale_inv
|
||||
|
||||
|
||||
def _make_ids(head_sizes: tuple[int, ...], num_tokens: int, seed: int) -> torch.Tensor:
|
||||
"""Ids of each head, drawn from that head's own bucket range."""
|
||||
torch.manual_seed(seed)
|
||||
offset = 0
|
||||
columns = []
|
||||
for size in head_sizes:
|
||||
columns.append(torch.randint(offset, offset + size, (num_tokens, 1)))
|
||||
offset += size
|
||||
return torch.cat(columns, dim=1).to(torch.int32).cuda()
|
||||
|
||||
|
||||
def _reference(weight, scale_inv, ids) -> torch.Tensor:
|
||||
"""Dequantized lookup against the whole table."""
|
||||
valid = ids >= 0
|
||||
safe_ids = ids.clamp_min(0).long()
|
||||
values = torch.nn.functional.embedding(safe_ids, weight).float()
|
||||
scales = torch.nn.functional.embedding(safe_ids, scale_inv)
|
||||
scales = (scales.to(torch.int32) << 23).view(torch.float32)
|
||||
values = values.unflatten(-1, (-1, BLOCK)) * scales.unsqueeze(-1)
|
||||
return values.flatten(-2).to(torch.bfloat16).masked_fill(~valid.unsqueeze(-1), 0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"architecture,options,edp_ranks,etp_ranks",
|
||||
[
|
||||
("DeepseekV41ForCausalLM", {}, [0, 2], [0, 1]),
|
||||
("Qwen4ExpForCausalLM", {}, None, [0, 1]),
|
||||
(
|
||||
"Qwen4ExpForCausalLM",
|
||||
{"embedding_across_dp": True},
|
||||
None,
|
||||
[0, 1, 2, 3],
|
||||
),
|
||||
("Qwen4ExpForConditionalGeneration", {}, None, [0, 1]),
|
||||
],
|
||||
)
|
||||
def test_engram_group_creation_honors_model_and_options(
|
||||
monkeypatch, architecture, options, edp_ranks, etp_ranks
|
||||
):
|
||||
config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(architecture=architecture, is_moe=False),
|
||||
parallel_config=ParallelConfig(
|
||||
tensor_parallel_size=2,
|
||||
data_parallel_size=2,
|
||||
distributed_executor_backend="mp",
|
||||
),
|
||||
engram_config=EngramConfig(**options),
|
||||
)
|
||||
for name in (
|
||||
"_TP",
|
||||
"_ETP",
|
||||
"_ENGRAM_DP",
|
||||
"_DCP",
|
||||
"_PCP",
|
||||
"_PP",
|
||||
"_DP",
|
||||
"_EP",
|
||||
"_EPLB",
|
||||
):
|
||||
monkeypatch.setattr(parallel_state, name, None)
|
||||
monkeypatch.setattr(parallel_state, "_WORLD", SimpleNamespace(local_rank=0))
|
||||
monkeypatch.setattr(torch.distributed, "is_initialized", lambda: True)
|
||||
monkeypatch.setattr(torch.distributed, "get_world_size", lambda: 4)
|
||||
monkeypatch.setattr(torch.distributed, "get_rank", lambda: 0)
|
||||
monkeypatch.setattr(parallel_state, "get_node_count", lambda: 1)
|
||||
|
||||
def make_group(group_ranks, *args, **kwargs):
|
||||
ranks = next(ranks for ranks in group_ranks if 0 in ranks)
|
||||
return SimpleNamespace(ranks=ranks, rank_in_group=0, world_size=len(ranks))
|
||||
|
||||
monkeypatch.setattr(parallel_state, "init_model_parallel_group", make_group)
|
||||
with set_current_vllm_config(config):
|
||||
initialize_model_parallel(tensor_model_parallel_size=2, backend="nccl")
|
||||
group = get_engram_dp_group()
|
||||
assert (group.ranks if group is not None else None) == edp_ranks
|
||||
assert parallel_state.get_etp_group().ranks == etp_ranks
|
||||
|
||||
|
||||
def _worker(rank: int, tp_size: int, port: int) -> None:
|
||||
torch.accelerator.set_device_index(rank)
|
||||
update_environment_variables(
|
||||
{
|
||||
"RANK": str(rank),
|
||||
"LOCAL_RANK": str(rank),
|
||||
"WORLD_SIZE": str(WORLD_SIZE),
|
||||
"MASTER_ADDR": "localhost",
|
||||
"MASTER_PORT": str(port),
|
||||
}
|
||||
)
|
||||
dp_size = WORLD_SIZE // tp_size
|
||||
dp_rank, tp_rank = divmod(rank, tp_size)
|
||||
vllm_config = VllmConfig(
|
||||
parallel_config=ParallelConfig(
|
||||
tensor_parallel_size=tp_size,
|
||||
data_parallel_size=dp_size,
|
||||
data_parallel_rank=dp_rank,
|
||||
),
|
||||
scheduler_config=SchedulerConfig(
|
||||
max_model_len=8,
|
||||
is_encoder_decoder=False,
|
||||
max_num_batched_tokens=8,
|
||||
max_num_seqs=8,
|
||||
),
|
||||
)
|
||||
# Supply the resolved Engram settings without constructing a full model.
|
||||
vllm_config.engram_config = EngramConfig()
|
||||
vllm_config.model_config = SimpleNamespace(
|
||||
architecture="DeepseekV41ForCausalLM",
|
||||
is_moe=True,
|
||||
sleep_mode_offload_cudagraph=False,
|
||||
)
|
||||
try:
|
||||
init_distributed_environment()
|
||||
with set_current_vllm_config(vllm_config):
|
||||
initialize_model_parallel(tensor_model_parallel_size=tp_size)
|
||||
assert get_engram_dp_size() == dp_size
|
||||
assert get_engram_dp_group().ranks == [
|
||||
tp_rank + replica * tp_size for replica in range(dp_size)
|
||||
]
|
||||
assert engram_head_shard_rank() == tp_rank * dp_size + dp_rank
|
||||
|
||||
for failure in ("create", "open", "mmap", "register", "pinned", None):
|
||||
_check_shared_storage_lifetime_and_failures(failure)
|
||||
|
||||
# Reuse the processes and communication groups across storage modes.
|
||||
for (cpu_offload, dp_shared_memory), n_heads, sequence_parallel in product(
|
||||
[(False, False), (True, False), (True, True)],
|
||||
[8, 7],
|
||||
[False, True] if tp_size > 1 else [False],
|
||||
):
|
||||
_check_table(
|
||||
vllm_config,
|
||||
cpu_offload,
|
||||
dp_shared_memory,
|
||||
n_heads,
|
||||
sequence_parallel,
|
||||
)
|
||||
if tp_size == 1:
|
||||
_check_fp8_batch(vllm_config)
|
||||
finally:
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
|
||||
def _check_fp8_batch(vllm_config):
|
||||
"""MXFP8 staging matches the tables across peer/all-to-all slots, idle
|
||||
replicas and graph replays."""
|
||||
from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphCapture
|
||||
|
||||
dp_rank = vllm_config.parallel_config.data_parallel_rank
|
||||
layers = []
|
||||
for index in range(2):
|
||||
weight, scale = _full_table(sum(HEAD_SIZES))
|
||||
weight = (weight.float() * (1 - 2 * index)).to(torch.float8_e4m3fn)
|
||||
table = ParallelEngramEmbedding(
|
||||
sum(HEAD_SIZES), DIM, HEAD_SIZES, cpu_offload=True
|
||||
)
|
||||
table.weight.weight_loader(table.weight, weight)
|
||||
table.weight_scale_inv.weight_loader(table.weight_scale_inv, scale)
|
||||
staged = torch.empty(0, device="cuda")
|
||||
reference = (weight.cuda(), scale.cuda())
|
||||
layers.append(SimpleNamespace(embed_tokens=table, staged_rows=staged))
|
||||
layers[-1].layer_hash_index, layers[-1].reference = index, reference
|
||||
batch = EngramBatch(layers, 257)
|
||||
|
||||
def check(ids):
|
||||
tokens = ids.shape[0]
|
||||
for layer in layers:
|
||||
x = batch.wkv_input(layer, tokens)
|
||||
sf = convert_swizzled_to_linear(x.scale, tokens, x.data.shape[1], BLOCK)
|
||||
actual = dequant_mxfp8_to_bf16(x.data, sf).unflatten(-1, (-1, DIM))
|
||||
weight, scale = layer.reference
|
||||
expected = _reference(weight, scale, ids[:, layer.layer_hash_index])
|
||||
torch.testing.assert_close(actual, expected, atol=0, rtol=0)
|
||||
|
||||
for counts in ((7, 0, 5, 1), (127,) * 4, (128,) * 4, (257, 0, 129, 1)):
|
||||
tokens = counts[dp_rank]
|
||||
seed = 700 + 10 * dp_rank
|
||||
ids = torch.stack([_make_ids(HEAD_SIZES, tokens, seed + i) for i in (0, 1)], 1)
|
||||
with set_forward_context(
|
||||
None,
|
||||
vllm_config,
|
||||
num_tokens=tokens,
|
||||
num_tokens_across_dp=torch.tensor(counts, dtype=torch.int32),
|
||||
):
|
||||
batch.prepare_embeddings(ids)
|
||||
check(ids)
|
||||
for capture in ("full", "breakable") if len(set(counts)) == 1 else ():
|
||||
warmup = torch.cuda.Stream()
|
||||
warmup.wait_stream(torch.cuda.current_stream())
|
||||
if capture == "full":
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph, stream=warmup):
|
||||
batch.prepare_embeddings(ids)
|
||||
else:
|
||||
graph = BreakableCUDAGraphCapture()
|
||||
with torch.cuda.stream(warmup), graph:
|
||||
batch.prepare_embeddings(ids)
|
||||
torch.cuda.current_stream().wait_stream(warmup)
|
||||
for step in (2, 3):
|
||||
replay_ids = _make_ids(HEAD_SIZES, tokens, seed + step)
|
||||
ids.copy_(torch.stack([replay_ids] * 2, 1))
|
||||
graph.replay()
|
||||
check(ids)
|
||||
del graph
|
||||
|
||||
|
||||
def _check_shared_storage_lifetime_and_failures(failure):
|
||||
"""One peer's failure must roll back every mapping/registration, without hangs."""
|
||||
import gc
|
||||
import weakref
|
||||
|
||||
group = get_engram_dp_group()
|
||||
runtime = torch.cuda.cudart()
|
||||
make_file = engram_ops.tempfile.NamedTemporaryFile
|
||||
map_file = engram_ops.mmap.mmap
|
||||
paths, mappings, registrations, unregistrations = [], [], [], []
|
||||
failed_rank = 0 if failure == "create" else 1
|
||||
|
||||
def fail():
|
||||
raise OSError(f"injected {failure} failure")
|
||||
|
||||
def temporary_file(*args, **kwargs):
|
||||
if failure == "create":
|
||||
fail()
|
||||
file = make_file(*args, **kwargs)
|
||||
paths.append(Path(file.name))
|
||||
return file
|
||||
|
||||
def open_file(*args, **kwargs):
|
||||
if failure == "open" and group.rank_in_group == failed_rank:
|
||||
fail()
|
||||
return open(*args, **kwargs)
|
||||
|
||||
def mapping(*args, **kwargs):
|
||||
if failure == "mmap" and group.rank_in_group == failed_rank:
|
||||
fail()
|
||||
result = map_file(*args, **kwargs)
|
||||
mappings.append(weakref.ref(result))
|
||||
return result
|
||||
|
||||
def register(pointer, size, flags):
|
||||
if failure == "register" and group.rank_in_group == failed_rank:
|
||||
return SimpleNamespace(value=1)
|
||||
result = runtime.cudaHostRegister(pointer, size, flags)
|
||||
assert result.value == 0
|
||||
registrations.append(pointer)
|
||||
return result
|
||||
|
||||
def unregister(pointer):
|
||||
unregistrations.append(pointer)
|
||||
result = runtime.cudaHostUnregister(pointer)
|
||||
assert result.value == 0
|
||||
return result
|
||||
|
||||
with pytest.MonkeyPatch.context() as patch:
|
||||
patch.setattr(
|
||||
engram_ops,
|
||||
"tempfile",
|
||||
SimpleNamespace(NamedTemporaryFile=temporary_file),
|
||||
)
|
||||
patch.setattr(engram_ops, "open", open_file, raising=False)
|
||||
patch.setattr(
|
||||
engram_ops,
|
||||
"mmap",
|
||||
SimpleNamespace(mmap=mapping, MAP_SHARED=engram_ops.mmap.MAP_SHARED),
|
||||
)
|
||||
patch.setattr(
|
||||
torch.cuda,
|
||||
"cudart",
|
||||
lambda: SimpleNamespace(
|
||||
cudaHostRegister=register,
|
||||
cudaHostUnregister=unregister,
|
||||
),
|
||||
)
|
||||
if failure == "pinned" and group.rank_in_group == failed_rank:
|
||||
patch.setattr(torch.Tensor, "is_pinned", lambda self: False)
|
||||
if failure is not None:
|
||||
stage = "cudaHostRegister" if failure in ("register", "pinned") else failure
|
||||
with pytest.raises(RuntimeError, match=f"EDP rank {failed_rank}.*{stage}"):
|
||||
engram_ops.DPSharedEngramStorage(128, DIM, BLOCK, group)
|
||||
assert unregistrations == registrations
|
||||
else:
|
||||
storage = engram_ops.DPSharedEngramStorage(128, DIM, BLOCK, group)
|
||||
assert all(not path.exists() for path in paths)
|
||||
storage_ref = weakref.ref(storage)
|
||||
parameter = torch.nn.Parameter(storage.weight, requires_grad=False)
|
||||
views = storage.get_views(parameter, storage.weight_scale_inv)
|
||||
del storage
|
||||
gc.collect()
|
||||
assert storage_ref() is None
|
||||
assert not unregistrations
|
||||
del parameter
|
||||
gc.collect()
|
||||
assert not unregistrations # UVA views still retain the CPU allocation.
|
||||
del views
|
||||
gc.collect()
|
||||
assert unregistrations == registrations
|
||||
for ref in mappings:
|
||||
mapped = ref()
|
||||
assert mapped is None or mapped.closed
|
||||
assert all(not path.exists() for path in paths)
|
||||
|
||||
|
||||
def _check_table(
|
||||
vllm_config, cpu_offload, dp_shared_memory, n_heads, sequence_parallel
|
||||
):
|
||||
parallel = vllm_config.parallel_config
|
||||
dp_size, tp_size = parallel.data_parallel_size, parallel.tensor_parallel_size
|
||||
dp_rank = parallel.data_parallel_rank
|
||||
tp_rank = engram_ops.get_tensor_model_parallel_rank()
|
||||
multithread = (
|
||||
dp_shared_memory and n_heads == 7 and sequence_parallel == (tp_size > 1)
|
||||
)
|
||||
load_config = (
|
||||
LoadConfig(
|
||||
load_format="safetensors",
|
||||
use_tqdm_on_load=False,
|
||||
model_loader_extra_config={
|
||||
"enable_multithread_load": True,
|
||||
"num_threads": 2,
|
||||
},
|
||||
)
|
||||
if multithread
|
||||
else vllm_config.load_config
|
||||
)
|
||||
with pytest.MonkeyPatch.context() as m:
|
||||
head_sizes = HEAD_SIZES[:n_heads]
|
||||
rows = sum(head_sizes)
|
||||
config = SimpleNamespace(
|
||||
hidden_size=DIM,
|
||||
hc_mult=2,
|
||||
rms_norm_eps=1e-6,
|
||||
engram_layer_ids=[0],
|
||||
engram_num_embeddings=[rows],
|
||||
engram_max_ngram_size=2,
|
||||
engram_n_heads=n_heads,
|
||||
engram_head_dim=DIM,
|
||||
engram_compressed_vocab_size=32,
|
||||
engram_pad_token_id=0,
|
||||
engram_vocab_size=HEAD_SIZES[0],
|
||||
)
|
||||
with set_current_vllm_config(vllm_config):
|
||||
weight, scale_inv = _full_table(rows)
|
||||
layout = EngramLayout(config)
|
||||
# Stub configuration inputs, not Engram's initialization or execution.
|
||||
component_config = SimpleNamespace(
|
||||
engram_config=EngramConfig(
|
||||
cpu_offload=cpu_offload,
|
||||
dp_shared_memory=dp_shared_memory,
|
||||
),
|
||||
scheduler_config=vllm_config.scheduler_config,
|
||||
load_config=load_config,
|
||||
)
|
||||
with m.context() as init_patch, torch.device("cuda"):
|
||||
init_patch.setattr(
|
||||
engram_ops, "get_current_vllm_config", lambda: component_config
|
||||
)
|
||||
engram = Engram(
|
||||
config,
|
||||
quant_config=None,
|
||||
layout=layout,
|
||||
layer_hash_index=0,
|
||||
use_sequence_parallel=sequence_parallel,
|
||||
prefix="model.layers.0.engram",
|
||||
)
|
||||
layer = engram.embed_tokens
|
||||
if multithread:
|
||||
_load_shared_multithread(layer, weight, scale_inv, load_config)
|
||||
else:
|
||||
# Only the leader has valid checkpoint payload in shared mode.
|
||||
loader_weight = weight if not dp_shared_memory or dp_rank == 0 else None
|
||||
loader_scale = (
|
||||
scale_inv if not dp_shared_memory or dp_rank == 0 else None
|
||||
)
|
||||
layer.weight.weight_loader(layer.weight, loader_weight)
|
||||
layer.weight_scale_inv.weight_loader(
|
||||
layer.weight_scale_inv, loader_scale
|
||||
)
|
||||
if dp_shared_memory:
|
||||
assert layer.dp_size == 1
|
||||
assert layer.head_start == tp_rank * layer.part_n_hash_cols
|
||||
assert layer.weight.is_pinned() and layer.weight_scale_inv.is_pinned()
|
||||
|
||||
def unexpected_collective(*args, **kwargs):
|
||||
raise AssertionError("shared lookup must not use DP collectives")
|
||||
|
||||
m.setattr(get_engram_dp_group(), "all_gather", unexpected_collective)
|
||||
|
||||
token_counts = _token_counts(dp_size)
|
||||
|
||||
# Production forward runs after the construction-time config has exited.
|
||||
assert get_current_vllm_config_or_none() is None
|
||||
for batch_idx, counts in enumerate(token_counts):
|
||||
num_tokens = counts[dp_rank]
|
||||
# Ids differ per replica but must agree across its TP ranks.
|
||||
ids = _make_ids(head_sizes, num_tokens, seed=100 + dp_rank)
|
||||
if batch_idx == 2 and dp_rank == 1:
|
||||
ids.fill_(engram_ops.DEAD_ID)
|
||||
with set_forward_context(
|
||||
None,
|
||||
vllm_config,
|
||||
num_tokens=num_tokens,
|
||||
num_tokens_across_dp=torch.tensor(counts, dtype=torch.int32),
|
||||
):
|
||||
gathered = gather_engram_hashes(
|
||||
ids, dp_shared_memory=layer.dp_shared_memory
|
||||
)
|
||||
expected_tokens = (
|
||||
num_tokens if dp_shared_memory else max(counts) * dp_size
|
||||
)
|
||||
assert gathered.shape[0] == expected_tokens
|
||||
engram.prepare_embeddings(gathered)
|
||||
looked_up = engram.embed(ids)
|
||||
standalone = layer(ids)
|
||||
expected = _reference(weight.cuda(), scale_inv.cuda(), ids)
|
||||
torch.testing.assert_close(standalone, expected, atol=0, rtol=0)
|
||||
if sequence_parallel:
|
||||
# SP hands back only this rank's token shard, zero-padded.
|
||||
chunk = -(-num_tokens // tp_size)
|
||||
expected = torch.nn.functional.pad(
|
||||
expected, (0, 0, 0, 0, 0, chunk * tp_size - num_tokens)
|
||||
)
|
||||
expected = expected[tp_rank * chunk : (tp_rank + 1) * chunk]
|
||||
torch.testing.assert_close(looked_up, expected, atol=0, rtol=0)
|
||||
|
||||
if (
|
||||
cpu_offload
|
||||
and n_heads == 7
|
||||
and not sequence_parallel
|
||||
and (not dp_shared_memory or tp_size == 1)
|
||||
):
|
||||
_check_dummy_hash_model_forward(
|
||||
vllm_config,
|
||||
engram,
|
||||
head_sizes,
|
||||
weight,
|
||||
scale_inv,
|
||||
dp_shared_memory,
|
||||
)
|
||||
|
||||
replay = n_heads == 7 and (
|
||||
(dp_shared_memory and sequence_parallel == (tp_size > 1))
|
||||
or (
|
||||
cpu_offload
|
||||
and not dp_shared_memory
|
||||
and sequence_parallel == (tp_size > 1)
|
||||
)
|
||||
)
|
||||
if replay:
|
||||
captures = ("full", "breakable")
|
||||
for capture in captures:
|
||||
_check_prefetch_replay(
|
||||
engram,
|
||||
head_sizes,
|
||||
weight,
|
||||
scale_inv,
|
||||
dp_shared_memory,
|
||||
capture,
|
||||
)
|
||||
if dp_shared_memory and n_heads == 7 and sequence_parallel == (tp_size > 1):
|
||||
with m.context() as storage_patch:
|
||||
storage_patch.setattr(layer.weight, "data", layer.weight.data.clone())
|
||||
with pytest.raises(RuntimeError, match="storage must not be replaced"):
|
||||
layer(ids)
|
||||
|
||||
|
||||
def _check_dummy_hash_model_forward(
|
||||
vllm_config, engram, head_sizes, weight, scales, dp_shared_memory
|
||||
):
|
||||
"""A metadata-less replica joins DP sharding but skips a shared lookup."""
|
||||
from vllm.models.deepseek_v41.common.engram import NgramHashState
|
||||
from vllm.models.deepseek_v41.nvidia import model as model_ops
|
||||
|
||||
dp_rank = vllm_config.parallel_config.data_parallel_rank
|
||||
if dp_shared_memory and dp_rank != 1:
|
||||
return
|
||||
tokens = 4
|
||||
ids = _make_ids(head_sizes, tokens, seed=300 + dp_rank)
|
||||
state = NgramHashState.__new__(NgramHashState)
|
||||
torch.nn.Module.__init__(state)
|
||||
state.multipliers = torch.empty(1, 2, dtype=torch.int64, device="cuda")
|
||||
state.primes = torch.empty(1, 1, len(head_sizes), dtype=torch.int64, device="cuda")
|
||||
state.ensure_cache = lambda: True
|
||||
# Hash arithmetic is covered separately; keep the real model's branch and
|
||||
# NgramHashState.dummy_hashes, plus real preparation/lookup collectives.
|
||||
state.forward = lambda *args: ids.unsqueeze(1)
|
||||
|
||||
class Decoder(SimpleNamespace):
|
||||
# Engram hashes and the mask stay the last positional arguments;
|
||||
# **kwargs absorbs the decoder's keyword-only flags.
|
||||
def __call__(self, hidden, positions, input_ids, *args, **kwargs):
|
||||
hashes, keep = args[-2:]
|
||||
if dp_shared_memory and dp_rank == 1:
|
||||
assert hashes is None and keep is None
|
||||
output = hidden.new_zeros(
|
||||
tokens, len(head_sizes), DIM, dtype=torch.bfloat16
|
||||
)
|
||||
else:
|
||||
assert hashes is not None and keep is not None
|
||||
assert bool(keep.all()) == (dp_rank != 1)
|
||||
if dp_rank == 1:
|
||||
assert torch.all(hashes == engram_ops.DEAD_ID)
|
||||
output = engram.embed(hashes[:, 0])
|
||||
# Trailing None is previous_aux; this stub captures no aux states.
|
||||
return output, hidden, hidden, hidden, hidden, None
|
||||
|
||||
model = model_ops.DeepseekV4Model.__new__(model_ops.DeepseekV4Model)
|
||||
torch.nn.Module.__init__(model)
|
||||
model.parallel_config = vllm_config.parallel_config
|
||||
model.use_native_mega_moe = False
|
||||
model.use_sequence_parallel = False
|
||||
model.fuse_mhc_all_reduce = False
|
||||
model.engram_batch = None
|
||||
model.engram_hash = state
|
||||
model.engram_swa_prefix = "swa"
|
||||
model.engram_dp_shared_memory = dp_shared_memory
|
||||
model.layers = [Decoder(engram=engram)]
|
||||
model.start_layer = 0
|
||||
model.end_layer = 1
|
||||
model.decoder_replay_start = model.end_layer
|
||||
model.decoder_replay_layers = None
|
||||
model.aux_hidden_state_layers = ()
|
||||
metadata = (
|
||||
None
|
||||
if dp_rank == 1
|
||||
else {
|
||||
"swa": SimpleNamespace(
|
||||
query_start_loc=None, slot_mapping=None, block_table=None
|
||||
)
|
||||
}
|
||||
)
|
||||
with pytest.MonkeyPatch.context() as patch:
|
||||
patch.setattr(
|
||||
model_ops,
|
||||
"get_pp_group",
|
||||
lambda: SimpleNamespace(
|
||||
is_first_rank=True,
|
||||
is_last_rank=False,
|
||||
),
|
||||
)
|
||||
patch.setattr(model_ops, "mhc_post_tilelang", lambda hidden, *args: hidden)
|
||||
with set_forward_context(
|
||||
metadata,
|
||||
vllm_config,
|
||||
num_tokens=tokens,
|
||||
num_tokens_across_dp=torch.full(
|
||||
(vllm_config.parallel_config.data_parallel_size,),
|
||||
tokens,
|
||||
dtype=torch.int32,
|
||||
),
|
||||
):
|
||||
output = model(
|
||||
torch.arange(tokens, device="cuda"),
|
||||
torch.arange(tokens, device="cuda"),
|
||||
intermediate_tensors=None,
|
||||
inputs_embeds=torch.zeros(tokens, DIM, device="cuda"),
|
||||
lookback_token_ids=torch.full((1, 1), -1, device="cuda"),
|
||||
)["hidden_states"]
|
||||
expected_ids = torch.full_like(ids, engram_ops.DEAD_ID) if dp_rank == 1 else ids
|
||||
expected = _reference(weight.cuda(), scales.cuda(), expected_ids)
|
||||
torch.testing.assert_close(output, expected, atol=0, rtol=0)
|
||||
|
||||
|
||||
def _load_shared_multithread(layer, weight, scale_inv, load_config):
|
||||
"""Opposite shard completion orders must leave every shared parameter ready."""
|
||||
from safetensors.torch import save_file
|
||||
|
||||
from vllm.model_executor.model_loader import weight_utils
|
||||
from vllm.model_executor.model_loader.default_loader import DefaultModelLoader
|
||||
|
||||
tensors = {
|
||||
"weight": weight,
|
||||
"weight_scale_inv": scale_inv.view(torch.float8_e8m0fnu),
|
||||
}
|
||||
order = list(tensors)
|
||||
if get_engram_dp_group().rank_in_group % 2:
|
||||
order.reverse()
|
||||
first_loaded = Event()
|
||||
original_load = weight_utils.load_file
|
||||
|
||||
def load_file(path, **kwargs):
|
||||
if Path(path).stem != order[0]:
|
||||
assert first_loaded.wait(timeout=30), "First shared weight load stalled"
|
||||
return original_load(path, **kwargs)
|
||||
|
||||
with TemporaryDirectory() as directory, pytest.MonkeyPatch.context() as m:
|
||||
for name, tensor in tensors.items():
|
||||
save_file({name: tensor}, str(Path(directory) / f"{name}.safetensors"))
|
||||
m.setattr(weight_utils, "load_file", load_file)
|
||||
loader = DefaultModelLoader(load_config)
|
||||
source = DefaultModelLoader.Source(directory, revision=None)
|
||||
loaded = []
|
||||
for name, tensor in loader._get_weights_iterator(source):
|
||||
param = getattr(layer, name)
|
||||
param.weight_loader(param, tensor)
|
||||
loaded.append(name)
|
||||
first_loaded.set()
|
||||
assert loaded == order
|
||||
|
||||
# Check before any further collective could mask incomplete shared writes.
|
||||
for name, tensor in tensors.items():
|
||||
param = getattr(layer, name)
|
||||
expected = tensor.narrow(0, param.engram_vocab_start, param.shape[0])
|
||||
torch.testing.assert_close(
|
||||
param.view(torch.uint8), expected.view(torch.uint8), atol=0, rtol=0
|
||||
)
|
||||
|
||||
|
||||
def _check_prefetch_replay(
|
||||
engram, head_sizes, weight, scale_inv, dp_shared_memory, capture
|
||||
):
|
||||
"""DP-sharded and shared prefetches must replay with new IDs."""
|
||||
from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphCapture
|
||||
|
||||
ids = _make_ids(head_sizes, 8, seed=501)
|
||||
tp_rank = engram_ops.get_tensor_model_parallel_rank()
|
||||
tokens = 8 // engram.embed_tokens.tp_size if engram.use_sequence_parallel else 8
|
||||
output = torch.empty(
|
||||
tokens, len(head_sizes), DIM, device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
|
||||
def step():
|
||||
gathered = gather_engram_hashes(ids.clone(), dp_shared_memory=dp_shared_memory)
|
||||
engram.prepare_embeddings(gathered)
|
||||
output.copy_(engram.embed(ids))
|
||||
|
||||
with pytest.MonkeyPatch.context() as patch:
|
||||
patch.setattr(engram_ops, "engram_gathered_num_tokens", lambda: 8)
|
||||
warmup = torch.cuda.Stream()
|
||||
warmup.wait_stream(torch.cuda.current_stream())
|
||||
with torch.cuda.stream(warmup):
|
||||
step()
|
||||
if capture == "breakable":
|
||||
graph = BreakableCUDAGraphCapture()
|
||||
with torch.cuda.stream(warmup), graph:
|
||||
step()
|
||||
assert graph.num_eager_breaks == 2
|
||||
else:
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph, stream=warmup):
|
||||
step()
|
||||
torch.cuda.current_stream().wait_stream(warmup)
|
||||
for iteration in range(2):
|
||||
ids.copy_(_make_ids(head_sizes, 8, seed=600 + iteration))
|
||||
graph.replay()
|
||||
expected = _reference(weight.cuda(), scale_inv.cuda(), ids)
|
||||
if engram.use_sequence_parallel:
|
||||
expected = expected[tp_rank * tokens : (tp_rank + 1) * tokens]
|
||||
torch.testing.assert_close(output, expected, atol=0, rtol=0)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA required")
|
||||
@pytest.mark.parametrize("tp_size", [1, 2])
|
||||
def test_engram_table_shared_across_dp_replicas(tp_size, monkeypatch):
|
||||
"""Check all storage modes with one process launch per TP/DP topology."""
|
||||
if torch.accelerator.device_count() < WORLD_SIZE:
|
||||
pytest.skip(f"Need {WORLD_SIZE} GPUs to run the test.")
|
||||
monkeypatch.setenv("VLLM_USE_BREAKABLE_CUDAGRAPH", "1")
|
||||
mp.spawn(_worker, args=(tp_size, get_open_port()), nprocs=WORLD_SIZE)
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"cpu_offload,dp_size,error",
|
||||
[
|
||||
(False, 4, "requires cpu_offload=True"),
|
||||
(True, 1, "effective Engram DP size is 1"),
|
||||
],
|
||||
)
|
||||
def test_engram_dp_shared_memory_runtime_requirements(
|
||||
monkeypatch, cpu_offload, dp_size, error
|
||||
):
|
||||
"""Report unmet storage/topology requirements before allocating CUDA memory."""
|
||||
monkeypatch.setattr(engram_ops, "get_engram_dp_size", lambda: dp_size)
|
||||
with pytest.raises(ValueError, match=error):
|
||||
ParallelEngramEmbedding(
|
||||
sum(HEAD_SIZES),
|
||||
DIM,
|
||||
HEAD_SIZES,
|
||||
cpu_offload=cpu_offload,
|
||||
dp_shared_memory=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("query_fails", [False, True])
|
||||
def test_engram_tables_too_large_for_shm_are_not_shared(monkeypatch, query_fails):
|
||||
"""A /dev/shm that is too small or cannot be queried must fall back, and the
|
||||
leader must still broadcast that decision so its peers do not hang."""
|
||||
broadcasts = []
|
||||
|
||||
def broadcast(obj):
|
||||
broadcasts.append(obj)
|
||||
return obj
|
||||
|
||||
def query(*args, **kwargs):
|
||||
raise OSError("statvfs failed")
|
||||
|
||||
monkeypatch.setattr(engram_ops, "get_engram_dp_size", lambda: 2)
|
||||
monkeypatch.setattr(
|
||||
engram_ops,
|
||||
"get_engram_dp_group",
|
||||
lambda: SimpleNamespace(rank_in_group=0, broadcast_object=broadcast),
|
||||
)
|
||||
if query_fails:
|
||||
monkeypatch.setattr(engram_ops, "check_shm_free_space", query)
|
||||
layout = SimpleNamespace(num_embeddings=(1 << 50,), head_dim=DIM)
|
||||
assert not engram_ops.can_share_engram_tables(layout)
|
||||
assert len(broadcasts) == 1 and broadcasts[0] is not None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"node_ids,dp_size,replica_size,expected",
|
||||
[
|
||||
([0] * 4, 1, 4, 1),
|
||||
([0] * 4, 4, 1, 4),
|
||||
([0] * 4, 2, 2, 2),
|
||||
([0] * 8 + [1] * 8, 8, 2, 4),
|
||||
([0] * 6 + [1] * 6, 4, 1, 2),
|
||||
([0] * 3 + [1] * 4, 7, 1, 1),
|
||||
([0] * 8 + [1] * 8 + [2] * 8, 8, 3, 1),
|
||||
([0, 1] * 4, 4, 2, 1),
|
||||
([0] * 3 + [1] * 5, 4, 2, 1),
|
||||
],
|
||||
)
|
||||
def test_engram_dp_shard_size_respects_node_boundaries(
|
||||
node_ids, dp_size, replica_size, expected, monkeypatch
|
||||
):
|
||||
"""Sharing must fall back when replicas or node rank ranges do not align."""
|
||||
monkeypatch.setattr(parallel_state, "get_node_count", lambda: len(set(node_ids)))
|
||||
monkeypatch.setattr(
|
||||
parallel_state, "get_world_group", lambda: SimpleNamespace(cpu_group=None)
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
parallel_state,
|
||||
"in_the_same_node_as",
|
||||
lambda group, src: [node == node_ids[src] for node in node_ids],
|
||||
)
|
||||
assert (
|
||||
parallel_state._engram_dp_shard_size(len(node_ids), dp_size, replica_size)
|
||||
== expected
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [0, 2, 4, 5])
|
||||
def test_engram_hash_padding_has_no_valid_rows(num_tokens, monkeypatch):
|
||||
"""Idle/padded slots must not query row zero; oversized batches are rejected."""
|
||||
monkeypatch.setattr(engram_ops, "engram_gathered_num_tokens", lambda: 4)
|
||||
monkeypatch.setattr(
|
||||
engram_ops,
|
||||
"get_engram_dp_group",
|
||||
lambda: SimpleNamespace(all_gather=lambda ids, dim: torch.cat([ids, ids], dim)),
|
||||
)
|
||||
ids = torch.full((num_tokens, 2, 3), 17, dtype=torch.int32)
|
||||
if num_tokens > 4:
|
||||
with pytest.raises(ValueError, match="exceeds the DP token slot"):
|
||||
gather_engram_hashes(ids)
|
||||
return
|
||||
gathered = gather_engram_hashes(ids).reshape(2, 4, 2, 3)
|
||||
for replica in gathered:
|
||||
torch.testing.assert_close(replica[:num_tokens], ids)
|
||||
assert torch.all(replica[num_tokens:] == engram_ops.DEAD_ID)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tp_size", [1, 2])
|
||||
@pytest.mark.parametrize("uniform", [False, True])
|
||||
def test_engram_hash_padding_ignores_other_nodes(tp_size, uniform, monkeypatch):
|
||||
"""Two four-GPU nodes pad only to their own maximum, including TP x DP."""
|
||||
group_size = 4 // tp_size
|
||||
counts = (
|
||||
[4] * (8 // tp_size)
|
||||
if uniform
|
||||
else [4096] + [2] * (group_size - 1) + [1, 3] + [0] * (group_size - 2)
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
engram_ops,
|
||||
"get_forward_context",
|
||||
lambda: SimpleNamespace(
|
||||
dp_metadata=SimpleNamespace(num_tokens_across_dp_cpu=torch.tensor(counts))
|
||||
),
|
||||
)
|
||||
for dp_rank in range(8 // tp_size):
|
||||
slot = 4 if uniform else (4096 if dp_rank < group_size else 3)
|
||||
ids = torch.full((counts[dp_rank], 2, 3), 17, dtype=torch.int32)
|
||||
monkeypatch.setattr(
|
||||
engram_ops,
|
||||
"get_dp_group",
|
||||
lambda rank=dp_rank: SimpleNamespace(rank_in_group=rank),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
engram_ops,
|
||||
"get_engram_dp_group",
|
||||
lambda rank=dp_rank: SimpleNamespace(
|
||||
rank_in_group=rank % group_size,
|
||||
world_size=group_size,
|
||||
all_gather=lambda tensor, dim: torch.cat([tensor] * group_size, dim),
|
||||
),
|
||||
)
|
||||
gathered = gather_engram_hashes(ids)
|
||||
expected = ids.new_full((slot, 2, 3), engram_ops.DEAD_ID)
|
||||
expected[: ids.shape[0]] = ids
|
||||
torch.testing.assert_close(
|
||||
gathered, expected.repeat(group_size, 1, 1), msg=f"DP rank {dp_rank}"
|
||||
)
|
||||
@@ -1,355 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import lm_eval
|
||||
import pytest
|
||||
import torch
|
||||
from transformers import DeepseekV4Config
|
||||
|
||||
from tests.utils import large_gpu_mark
|
||||
from vllm.distributed.eplb.eplb_state import EplbState
|
||||
from vllm.models.deepseek_v4.common.eplb_util import dspark_draft_supports_eplb
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.worker.gpu.eplb_utils import (
|
||||
EPLBController,
|
||||
draft_model_supports_eplb,
|
||||
)
|
||||
|
||||
|
||||
def get_model_args(
|
||||
model_name: str,
|
||||
spec_model_name: str | None,
|
||||
spec_method: str,
|
||||
tp_size: int,
|
||||
model_max_len: int,
|
||||
num_speculative_tokens: int = 1,
|
||||
use_async: bool = True,
|
||||
) -> dict:
|
||||
speculative_config = {
|
||||
"method": spec_method,
|
||||
"model": spec_model_name,
|
||||
"num_speculative_tokens": num_speculative_tokens,
|
||||
"max_model_len": model_max_len,
|
||||
}
|
||||
if spec_method == "dspark":
|
||||
speculative_config["model"] = model_name
|
||||
speculative_config["draft_sample_method"] = "probabilistic"
|
||||
eplb_config = {
|
||||
"num_redundant_experts": tp_size,
|
||||
"window_size": 128,
|
||||
"step_interval": 1024,
|
||||
"log_balancedness": False,
|
||||
"use_async": use_async,
|
||||
}
|
||||
model_args = {
|
||||
"pretrained": model_name,
|
||||
"dtype": "auto",
|
||||
"add_bos_token": True,
|
||||
"tensor_parallel_size": tp_size,
|
||||
"gpu_memory_utilization": 0.7,
|
||||
"speculative_config": speculative_config,
|
||||
"enable_expert_parallel": True,
|
||||
"eplb_config": eplb_config,
|
||||
"enable_eplb": True,
|
||||
"max_model_len": model_max_len,
|
||||
}
|
||||
if spec_method == "dspark":
|
||||
model_args["trust_remote_code"] = True
|
||||
return model_args
|
||||
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
current_platform.is_rocm(),
|
||||
reason="EPLB with Spec Decode is a work in progress on ROCm.",
|
||||
)
|
||||
|
||||
|
||||
class FakeEplbState:
|
||||
instances: list[FakeEplbState] = []
|
||||
|
||||
def __init__(self, parallel_config, device: torch.device):
|
||||
self.add_model_calls: list[tuple[object, object, str | None]] = []
|
||||
FakeEplbState.instances.append(self)
|
||||
|
||||
def add_model(
|
||||
self,
|
||||
model: object,
|
||||
model_config: object,
|
||||
*,
|
||||
model_name: str | None = None,
|
||||
) -> None:
|
||||
self.add_model_calls.append((model, model_config, model_name))
|
||||
|
||||
|
||||
def _make_dsv4_dspark_hf_config() -> DeepseekV4Config:
|
||||
return DeepseekV4Config(
|
||||
architectures=["DeepseekV4ForCausalLM"],
|
||||
hidden_size=128,
|
||||
num_hidden_layers=2,
|
||||
n_routed_experts=8,
|
||||
num_experts_per_tok=2,
|
||||
num_hash_layers=0,
|
||||
n_shared_experts=1,
|
||||
moe_intermediate_size=128,
|
||||
hc_mult=1,
|
||||
hc_eps=1e-5,
|
||||
rms_norm_eps=1e-5,
|
||||
dspark_target_layer_ids=[0],
|
||||
dspark_markov_rank=8,
|
||||
index_topk=4,
|
||||
head_dim=64,
|
||||
num_attention_heads=4,
|
||||
vocab_size=256,
|
||||
n_mtp_layers=2,
|
||||
enable_confidence_head=False,
|
||||
layer_types=["sliding_attention"] * 2,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def dspark_vllm_config():
|
||||
hf_config = _make_dsv4_dspark_hf_config()
|
||||
model_config = SimpleNamespace(
|
||||
dtype=torch.bfloat16, hf_config=hf_config, model="dspark"
|
||||
)
|
||||
return SimpleNamespace(
|
||||
model_config=model_config,
|
||||
quant_config=None,
|
||||
kernel_config=SimpleNamespace(moe_backend="deep_gemm_mega_moe"),
|
||||
parallel_config=SimpleNamespace(
|
||||
pipeline_parallel_size=1,
|
||||
tensor_parallel_size=1,
|
||||
data_parallel_size=1,
|
||||
enable_expert_parallel=True,
|
||||
enable_eplb=True,
|
||||
enable_elastic_ep=False,
|
||||
eplb_config=SimpleNamespace(num_redundant_experts=4),
|
||||
),
|
||||
scheduler_config=SimpleNamespace(max_num_batched_tokens=4),
|
||||
compilation_config=SimpleNamespace(static_forward_context={}),
|
||||
speculative_config=SimpleNamespace(
|
||||
method="dspark",
|
||||
draft_model_config=model_config,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _make_moe_topology(
|
||||
*,
|
||||
num_routed_experts: int = 8,
|
||||
num_redundant_experts: int = 4,
|
||||
num_expert_groups: int = 1,
|
||||
) -> SimpleNamespace:
|
||||
num_physical_experts = num_routed_experts + num_redundant_experts
|
||||
return SimpleNamespace(
|
||||
num_routed_experts=num_routed_experts,
|
||||
num_redundant_experts=num_redundant_experts,
|
||||
num_physical_experts=num_physical_experts,
|
||||
num_logical_experts=num_routed_experts,
|
||||
num_expert_groups=num_expert_groups,
|
||||
)
|
||||
|
||||
|
||||
def test_eplb_state_accepts_matching_dsv4_draft_and_target():
|
||||
draft = _make_moe_topology()
|
||||
target = _make_moe_topology()
|
||||
|
||||
EplbState.assert_confs_equal(draft, target)
|
||||
|
||||
|
||||
def test_eplb_state_rejects_mismatched_dsv4_draft_redundant_experts():
|
||||
draft = _make_moe_topology(num_redundant_experts=0)
|
||||
target = _make_moe_topology(num_redundant_experts=4)
|
||||
|
||||
with pytest.raises(RuntimeError, match="mismatch"):
|
||||
EplbState.assert_confs_equal(draft, target)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model_type", "expected"),
|
||||
[
|
||||
("deepseek_v4", True),
|
||||
("deepseek_v41", False),
|
||||
],
|
||||
)
|
||||
def test_dspark_draft_supports_eplb(
|
||||
dspark_vllm_config, model_type: str, expected: bool
|
||||
):
|
||||
draft_model_config = dspark_vllm_config.speculative_config.draft_model_config
|
||||
draft_model_config.hf_config.model_type = model_type
|
||||
assert dspark_draft_supports_eplb(draft_model_config) is expected
|
||||
|
||||
|
||||
def test_draft_model_supports_eplb_requires_draft_moe(dspark_vllm_config):
|
||||
assert not draft_model_supports_eplb(
|
||||
dspark_vllm_config.speculative_config,
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_type,load_dummy_weights,expected_registered",
|
||||
[
|
||||
("deepseek_v4", False, True),
|
||||
("deepseek_v4", True, False),
|
||||
("deepseek_v41", False, False),
|
||||
],
|
||||
)
|
||||
def test_eplb_dspark_draft_registration(
|
||||
dspark_vllm_config,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
model_type,
|
||||
load_dummy_weights: bool,
|
||||
expected_registered: bool,
|
||||
):
|
||||
"""Register V4 drafts; skip incompatible V4.1 drafts and dummy weights."""
|
||||
FakeEplbState.instances.clear()
|
||||
monkeypatch.setattr(
|
||||
"vllm.v1.worker.gpu.eplb_utils.EplbState",
|
||||
FakeEplbState,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.v1.worker.gpu.eplb_utils.get_mixture_of_experts_model",
|
||||
lambda model: model,
|
||||
)
|
||||
|
||||
draft_model_config = dspark_vllm_config.speculative_config.draft_model_config
|
||||
draft_model_config.hf_config.model_type = model_type
|
||||
draft = SimpleNamespace()
|
||||
controller = EPLBController(dspark_vllm_config.parallel_config, torch.device("cpu"))
|
||||
controller.prepare_load()
|
||||
speculator = SimpleNamespace(model=draft, eplb_state=None)
|
||||
|
||||
def set_eplb_state(state) -> None:
|
||||
speculator.eplb_state = state
|
||||
|
||||
speculator.set_eplb_state = set_eplb_state
|
||||
|
||||
registered = controller.maybe_register_speculator(
|
||||
speculator,
|
||||
dspark_vllm_config.speculative_config,
|
||||
load_dummy_weights=load_dummy_weights,
|
||||
)
|
||||
|
||||
assert registered is expected_registered
|
||||
assert controller.state is not None
|
||||
if expected_registered:
|
||||
assert controller.state.add_model_calls == [
|
||||
(draft, draft_model_config, "dspark (draft)")
|
||||
]
|
||||
assert speculator.eplb_state is controller.state
|
||||
else:
|
||||
assert controller.state.add_model_calls == []
|
||||
assert speculator.eplb_state is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_setup",
|
||||
[
|
||||
pytest.param(
|
||||
("mtp", "Qwen/Qwen3-Next-80B-A3B-Instruct", None, 4, 0.86, 1),
|
||||
marks=large_gpu_mark(min_gb=80),
|
||||
),
|
||||
pytest.param(
|
||||
(
|
||||
"eagle",
|
||||
"meta-llama/Llama-4-Scout-17B-16E-Instruct",
|
||||
"morgendave/EAGLE-Llama-4-Scout-17B-16E-Instruct",
|
||||
4,
|
||||
0.92,
|
||||
1,
|
||||
),
|
||||
marks=pytest.mark.skip(reason="Skipping due to CI OOM issues"),
|
||||
),
|
||||
pytest.param(
|
||||
(
|
||||
"dspark",
|
||||
"deepseek-ai/DeepSeek-V4-Flash-DSpark",
|
||||
None,
|
||||
4,
|
||||
0.95,
|
||||
7,
|
||||
),
|
||||
marks=large_gpu_mark(min_gb=80),
|
||||
),
|
||||
],
|
||||
ids=["qwen3_next_mtp", "llama4_eagle", "dsv4_dspark"],
|
||||
)
|
||||
def test_eplb_spec_decode(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
model_setup: tuple[str, str, str | None, int, float, int],
|
||||
):
|
||||
"""Test the correctness of EPLB speculative decoding with GSM8K dataset.
|
||||
|
||||
Applicable to MoE models with mtp, eagle, or dspark spec decode.
|
||||
"""
|
||||
(
|
||||
method,
|
||||
model_name,
|
||||
spec_model_name,
|
||||
tp_size,
|
||||
expected_gsm8k_value,
|
||||
num_speculative_tokens,
|
||||
) = model_setup
|
||||
if method == "dspark" and not current_platform.is_device_capability_family(100):
|
||||
pytest.skip("DSV4 DSpark EPLB requires SM100 for MegaMoE")
|
||||
|
||||
TASK = "gsm8k"
|
||||
FILTER = "exact_match,strict-match"
|
||||
RTOL = 0.03
|
||||
|
||||
model_args = get_model_args(
|
||||
model_name=model_name,
|
||||
spec_model_name=spec_model_name,
|
||||
spec_method=method,
|
||||
tp_size=tp_size,
|
||||
model_max_len=4096,
|
||||
num_speculative_tokens=num_speculative_tokens,
|
||||
)
|
||||
|
||||
results = lm_eval.simple_evaluate(
|
||||
model="vllm",
|
||||
model_args=model_args,
|
||||
tasks=TASK,
|
||||
batch_size=64,
|
||||
num_fewshot=8,
|
||||
)
|
||||
measured_value = results["results"][TASK][FILTER]
|
||||
assert (
|
||||
measured_value - RTOL < expected_gsm8k_value
|
||||
and measured_value + RTOL > expected_gsm8k_value
|
||||
), f"Expected: {expected_gsm8k_value} | Measured: {measured_value}"
|
||||
|
||||
|
||||
@large_gpu_mark(min_gb=80)
|
||||
def test_eplb_spec_decode_qwen3_next_mtp_async() -> None:
|
||||
"""Ensure async EPLB works with MTP speculative decoding for Qwen3-Next."""
|
||||
TASK = "gsm8k"
|
||||
FILTER = "exact_match,strict-match"
|
||||
RTOL = 0.03
|
||||
expected_gsm8k_value = 0.86
|
||||
|
||||
model_args = get_model_args(
|
||||
model_name="Qwen/Qwen3-Next-80B-A3B-Instruct",
|
||||
spec_model_name=None,
|
||||
spec_method="mtp",
|
||||
tp_size=4,
|
||||
model_max_len=4096,
|
||||
use_async=True,
|
||||
)
|
||||
|
||||
results = lm_eval.simple_evaluate(
|
||||
model="vllm",
|
||||
model_args=model_args,
|
||||
tasks=TASK,
|
||||
batch_size=64,
|
||||
num_fewshot=8,
|
||||
)
|
||||
measured_value = results["results"][TASK][FILTER]
|
||||
assert (
|
||||
measured_value - RTOL < expected_gsm8k_value
|
||||
and measured_value + RTOL > expected_gsm8k_value
|
||||
), f"Expected: {expected_gsm8k_value} | Measured: {measured_value}"
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,81 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import weakref
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm import LLM, PoolingParams
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
|
||||
MODEL_NAME = "intfloat/multilingual-e5-small"
|
||||
|
||||
PROMPTS = [
|
||||
"Hello, my name is",
|
||||
"The president of the United States is",
|
||||
"The capital of France is",
|
||||
"The future of AI is",
|
||||
]
|
||||
|
||||
TOKEN_IDS = [
|
||||
# Using ID={0, 1, 2, 3} results in NaN values,
|
||||
# so we add this offset of 1000
|
||||
[1000],
|
||||
[1000, 1001],
|
||||
[1000, 1002, 1001],
|
||||
[1000, 1003, 1001, 1002],
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def llm(vllm_runner):
|
||||
with vllm_runner(
|
||||
MODEL_NAME,
|
||||
max_model_len=None,
|
||||
max_num_batched_tokens=32768,
|
||||
tensor_parallel_size=1,
|
||||
gpu_memory_utilization=0.75,
|
||||
enforce_eager=True,
|
||||
seed=0,
|
||||
enable_chunked_prefill=None,
|
||||
) as runner:
|
||||
# pytest caches yielded fixtures until after teardown, so use a proxy to
|
||||
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
|
||||
yield weakref.proxy(runner.llm)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_multiple_pooling_params(llm: LLM):
|
||||
pooling_params = [
|
||||
PoolingParams(),
|
||||
PoolingParams(),
|
||||
PoolingParams(),
|
||||
PoolingParams(),
|
||||
]
|
||||
|
||||
# Multiple PoolingParams should be matched with each prompt
|
||||
outputs = llm.encode(PROMPTS, pooling_params=pooling_params, pooling_task="embed")
|
||||
assert len(PROMPTS) == len(outputs)
|
||||
|
||||
# Exception raised, if the size of params does not match the size of prompts
|
||||
with pytest.raises(VLLMValidationError):
|
||||
outputs = llm.encode(
|
||||
PROMPTS, pooling_params=pooling_params[:3], pooling_task="embed"
|
||||
)
|
||||
|
||||
# Single PoolingParams should be applied to every prompt
|
||||
single_pooling_params = PoolingParams()
|
||||
outputs = llm.encode(
|
||||
PROMPTS, pooling_params=single_pooling_params, pooling_task="embed"
|
||||
)
|
||||
assert len(PROMPTS) == len(outputs)
|
||||
|
||||
# pooling_params is None, default params should be applied
|
||||
outputs = llm.encode(PROMPTS, pooling_params=None, pooling_task="embed")
|
||||
assert len(PROMPTS) == len(outputs)
|
||||
|
||||
|
||||
def test_right_side_truncation(llm: LLM):
|
||||
# Embeddings models should truncate the end of the prompt
|
||||
tokenizer = llm.get_tokenizer()
|
||||
assert tokenizer.truncation_side == "right"
|
||||
@@ -1,102 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import weakref
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm import PoolingParams
|
||||
|
||||
MODEL_NAME = "intfloat/multilingual-e5-small"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def llm(vllm_runner):
|
||||
with vllm_runner(
|
||||
MODEL_NAME,
|
||||
max_model_len=None,
|
||||
max_num_seqs=2, # small to trigger tiling
|
||||
tensor_parallel_size=1,
|
||||
gpu_memory_utilization=0.75,
|
||||
enforce_eager=True,
|
||||
seed=0,
|
||||
enable_chunked_prefill=None,
|
||||
) as runner:
|
||||
# pytest caches yielded fixtures until after teardown, so use a proxy to
|
||||
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
|
||||
yield weakref.proxy(runner.llm)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_tiling_engine_basic(llm):
|
||||
"""Basic test with a small number of prompts (less than max_num_seqs).
|
||||
No tiling should be triggered, but the engine still processes correctly.
|
||||
"""
|
||||
prompts = ["Hello", "World"]
|
||||
outputs = llm.encode(prompts, pooling_task="embed")
|
||||
assert len(outputs) == len(prompts)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_tiling_engine_many_requests(llm):
|
||||
"""Test with a large number of prompts that exceeds max_num_seqs.
|
||||
This verifies that _run_tiling_engine correctly chunks requests,
|
||||
processes all of them, and returns outputs in the correct order.
|
||||
"""
|
||||
num_prompts = 10
|
||||
prompts = [f"Prompt {i}" for i in range(num_prompts)]
|
||||
outputs = llm.encode(prompts, pooling_task="embed")
|
||||
assert len(outputs) == num_prompts
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_tiling_engine_with_pooling_params(llm):
|
||||
"""Test the tiling engine when different PoolingParams are provided.
|
||||
The engine must handle a list of params that matches the number of prompts.
|
||||
"""
|
||||
num_prompts = 10
|
||||
prompts = [f"Prompt {i}" for i in range(num_prompts)]
|
||||
pooling_params = [PoolingParams() for _ in range(num_prompts)]
|
||||
|
||||
outputs = llm.encode(prompts, pooling_params=pooling_params, pooling_task="embed")
|
||||
assert len(outputs) == num_prompts
|
||||
|
||||
# Single PoolingParams shared across all prompts
|
||||
single_param = PoolingParams()
|
||||
outputs = llm.encode(prompts, pooling_params=single_param, pooling_task="embed")
|
||||
assert len(outputs) == num_prompts
|
||||
|
||||
# None PoolingParams should fall back to default
|
||||
outputs = llm.encode(prompts, pooling_params=None, pooling_task="embed")
|
||||
assert len(outputs) == num_prompts
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_tiling_engine_abort_on_exception(llm):
|
||||
"""Test that abort_request IS called with the correct arguments when an
|
||||
exception occurs inside the engine's step() loop.
|
||||
"""
|
||||
prompts = ["Prompt 0", "Prompt 1", "Prompt 2"]
|
||||
|
||||
# Mock the step method to throw an exception on the second call
|
||||
original_step = llm.llm_engine.step
|
||||
call_count = 0
|
||||
|
||||
def mocked_step():
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
if call_count == 2:
|
||||
raise RuntimeError("Simulated engine error")
|
||||
return original_step()
|
||||
|
||||
with mock.patch.object(llm.llm_engine, "step", side_effect=mocked_step):
|
||||
# We expect an exception to be raised from encode
|
||||
with mock.patch.object(llm.llm_engine, "abort_request") as mock_abort: # noqa: SIM117
|
||||
with pytest.raises(RuntimeError, match="Simulated engine error"):
|
||||
llm.encode(prompts, pooling_task="embed")
|
||||
|
||||
args, kwargs = mock_abort.call_args
|
||||
request_ids = args[0]
|
||||
assert isinstance(request_ids, list)
|
||||
assert len(request_ids) > 0
|
||||
@@ -1,99 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import Any
|
||||
|
||||
import openai
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
|
||||
MODEL_NAME = "sentence-transformers/all-MiniLM-L12-v2"
|
||||
max_model_len = 128
|
||||
|
||||
input = """Immerse yourself in the enchanting chronicle of calculus, a
|
||||
mathematical domain that has radically transformed our comprehension of
|
||||
change and motion. Despite its roots in ancient civilizations, the
|
||||
formal birth of calculus predominantly occurred in the 17th century,
|
||||
primarily under the influential guidance of Sir Isaac Newton and Gottfried
|
||||
Wilhelm Leibniz. The earliest traces of calculus concepts are found in
|
||||
ancient Greek mathematics,most notably in the works of Eudoxus and
|
||||
Archimedes, around 300 BCE. They utilized the 'method of exhaustion'—a
|
||||
technique for computing areas and volumes through the use of finite sums.
|
||||
This methodology laid crucial foundational work for integral calculus.
|
||||
In the 17th century, both Newton and Leibniz independently pioneered
|
||||
calculus, each contributing unique perspectives that would shape this new
|
||||
field."""
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
args = [
|
||||
"--runner",
|
||||
"pooling",
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--enforce-eager",
|
||||
"--max-model-len",
|
||||
str(max_model_len),
|
||||
]
|
||||
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def client(server):
|
||||
async with server.get_async_client() as async_client:
|
||||
yield async_client
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_smaller_truncation_size(client: openai.AsyncOpenAI):
|
||||
truncation_size = 10
|
||||
kwargs: dict[str, Any] = {
|
||||
"model": MODEL_NAME,
|
||||
"input": input,
|
||||
"truncate_prompt_tokens": truncation_size,
|
||||
}
|
||||
|
||||
response = await client.post(path="embeddings", cast_to=object, body={**kwargs})
|
||||
|
||||
assert response["usage"]["prompt_tokens"] == truncation_size
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bigger_truncation_size(client: openai.AsyncOpenAI):
|
||||
truncation_size = max_model_len + 1
|
||||
kwargs: dict[str, Any] = {
|
||||
"model": MODEL_NAME,
|
||||
"input": input,
|
||||
"truncate_prompt_tokens": truncation_size,
|
||||
}
|
||||
|
||||
with pytest.raises(openai.BadRequestError) as err:
|
||||
await client.post(path="embeddings", cast_to=object, body={**kwargs})
|
||||
|
||||
assert err.value.status_code == 400
|
||||
error_details = err.value.response.json()["error"]
|
||||
assert error_details["type"] == "BadRequestError"
|
||||
expected_message = (
|
||||
"truncate_prompt_tokens value is "
|
||||
"greater than max_model_len."
|
||||
" Please request a smaller truncation size."
|
||||
)
|
||||
assert error_details["message"] == expected_message
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_max_truncation_size(client: openai.AsyncOpenAI):
|
||||
truncation_size = -1
|
||||
kwargs: dict[str, Any] = {
|
||||
"model": MODEL_NAME,
|
||||
"input": input,
|
||||
"truncate_prompt_tokens": truncation_size,
|
||||
}
|
||||
|
||||
response = await client.post(path="embeddings", cast_to=object, body={**kwargs})
|
||||
|
||||
assert response["usage"]["prompt_tokens"] == max_model_len
|
||||
@@ -1,107 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import weakref
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.models.utils import softmax
|
||||
from vllm import LLM, ClassificationRequestOutput, PoolingParams
|
||||
from vllm.inputs import PromptType
|
||||
from vllm.tasks import PoolingTask
|
||||
|
||||
MODEL_NAME = "jason9693/Qwen2.5-1.5B-apeach"
|
||||
|
||||
prompt = "The chef prepared a delicious meal."
|
||||
prompt_token_ids = [785, 29706, 10030, 264, 17923, 15145, 13]
|
||||
num_labels = 2
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def llm(vllm_runner):
|
||||
with vllm_runner(
|
||||
MODEL_NAME,
|
||||
max_model_len=None,
|
||||
max_num_batched_tokens=32768,
|
||||
tensor_parallel_size=1,
|
||||
gpu_memory_utilization=0.75,
|
||||
enforce_eager=True,
|
||||
seed=0,
|
||||
enable_chunked_prefill=None,
|
||||
) as runner:
|
||||
# pytest caches yielded fixtures until after teardown, so use a proxy to
|
||||
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
|
||||
yield weakref.proxy(runner.llm)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_str_prompts(llm: LLM):
|
||||
outputs = llm.classify(prompt, use_tqdm=False)
|
||||
assert len(outputs) == 1
|
||||
assert isinstance(outputs[0], ClassificationRequestOutput)
|
||||
assert outputs[0].prompt_token_ids == prompt_token_ids
|
||||
assert len(outputs[0].outputs.probs) == num_labels
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_token_ids_prompts(llm: LLM):
|
||||
outputs = llm.classify([prompt_token_ids], use_tqdm=False)
|
||||
assert len(outputs) == 1
|
||||
assert isinstance(outputs[0], ClassificationRequestOutput)
|
||||
assert outputs[0].prompt_token_ids == prompt_token_ids
|
||||
assert len(outputs[0].outputs.probs) == num_labels
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_list_prompts(llm: LLM):
|
||||
prompts: list[PromptType] = [prompt, prompt_token_ids]
|
||||
outputs = llm.classify(prompts, use_tqdm=False)
|
||||
assert len(outputs) == 2
|
||||
for i in range(len(outputs)):
|
||||
assert isinstance(outputs[i], ClassificationRequestOutput)
|
||||
assert outputs[i].prompt_token_ids == prompt_token_ids
|
||||
assert len(outputs[i].outputs.probs) == num_labels
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_pooling_params(llm: LLM):
|
||||
def get_outputs(use_activation):
|
||||
outputs = llm.classify(
|
||||
prompt,
|
||||
pooling_params=PoolingParams(use_activation=use_activation),
|
||||
use_tqdm=False,
|
||||
)
|
||||
return torch.tensor([x.outputs.probs for x in outputs])
|
||||
|
||||
default = get_outputs(use_activation=None)
|
||||
w_activation = get_outputs(use_activation=True)
|
||||
wo_activation = get_outputs(use_activation=False)
|
||||
|
||||
assert torch.allclose(default, w_activation, atol=1e-2), (
|
||||
"Default should use activation."
|
||||
)
|
||||
assert not torch.allclose(w_activation, wo_activation, atol=1e-2), (
|
||||
"wo_activation should not use activation."
|
||||
)
|
||||
assert torch.allclose(softmax(wo_activation), w_activation, atol=1e-2), (
|
||||
"w_activation should be close to activation(wo_activation)."
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_score_api(llm: LLM):
|
||||
err_msg = "Scoring API is only enabled for num_labels == 1."
|
||||
with pytest.raises(ValueError, match=err_msg):
|
||||
llm.score("ping", "pong", use_tqdm=False)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("task", ["embed", "token_embed", "token_classify", "plugin"])
|
||||
def test_unsupported_tasks(llm: LLM, task: PoolingTask):
|
||||
if task == "plugin":
|
||||
err_msg = "No IOProcessor plugin installed."
|
||||
elif task == "token_classify":
|
||||
err_msg = "Try switching the model's pooling_task via.+"
|
||||
else:
|
||||
err_msg = "Embedding API is not supported by this model.+"
|
||||
with pytest.raises(ValueError, match=err_msg):
|
||||
llm.encode(prompt, pooling_task=task, use_tqdm=False)
|
||||
@@ -1,460 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.entrypoints.pooling.classify.protocol import ClassificationResponse
|
||||
from vllm.entrypoints.pooling.pooling.protocol import PoolingResponse
|
||||
|
||||
MODEL_NAME = "jason9693/Qwen2.5-1.5B-apeach"
|
||||
DTYPE = "float32" # Use float32 to avoid NaN issue
|
||||
input_text = "This product was excellent and exceeded my expectations"
|
||||
input_tokens = [1986, 1985, 572, 9073, 323, 33808, 847, 16665]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
args = [
|
||||
"--enforce-eager",
|
||||
"--max-model-len",
|
||||
"512",
|
||||
"--dtype",
|
||||
DTYPE,
|
||||
]
|
||||
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
def test_basic(server: RemoteOpenAIServer, model_name: str):
|
||||
# test /v1/models
|
||||
response = requests.get(server.url_for("/v1/models"))
|
||||
served_model = response.json()["data"][0]["id"]
|
||||
assert served_model == MODEL_NAME
|
||||
|
||||
# test /tokenize
|
||||
response = requests.post(
|
||||
server.url_for("/tokenize"),
|
||||
json={"model": model_name, "prompt": input_text},
|
||||
)
|
||||
assert response.json()["tokens"] == input_tokens
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
def test_completion_request(server: RemoteOpenAIServer, model_name: str):
|
||||
# test input: str
|
||||
classification_response = requests.post(
|
||||
server.url_for("classify"),
|
||||
json={"model": model_name, "input": input_text},
|
||||
)
|
||||
|
||||
classification_response.raise_for_status()
|
||||
output = ClassificationResponse.model_validate(classification_response.json())
|
||||
|
||||
assert output.object == "list"
|
||||
assert output.model == MODEL_NAME
|
||||
assert len(output.data) == 1
|
||||
assert hasattr(output.data[0], "label")
|
||||
assert hasattr(output.data[0], "probs")
|
||||
|
||||
# test input: list[int]
|
||||
classification_response = requests.post(
|
||||
server.url_for("classify"),
|
||||
json={"model": model_name, "input": input_tokens},
|
||||
)
|
||||
|
||||
classification_response.raise_for_status()
|
||||
output = ClassificationResponse.model_validate(classification_response.json())
|
||||
|
||||
assert output.object == "list"
|
||||
assert output.model == MODEL_NAME
|
||||
assert len(output.data) == 1
|
||||
assert hasattr(output.data[0], "label")
|
||||
assert hasattr(output.data[0], "probs")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
def test_completion_request_batched(server: RemoteOpenAIServer, model_name: str):
|
||||
N = 10
|
||||
|
||||
# test input: list[str]
|
||||
classification_response = requests.post(
|
||||
server.url_for("classify"),
|
||||
json={"model": model_name, "input": [input_text] * N},
|
||||
)
|
||||
output = ClassificationResponse.model_validate(classification_response.json())
|
||||
|
||||
assert len(output.data) == N
|
||||
for i, item in enumerate(output.data):
|
||||
assert item.index == i
|
||||
assert hasattr(item, "label")
|
||||
assert hasattr(item, "probs")
|
||||
assert len(item.probs) == item.num_classes
|
||||
assert item.label in ["Default", "Spoiled"]
|
||||
|
||||
# test input: list[list[int]]
|
||||
classification_response = requests.post(
|
||||
server.url_for("classify"),
|
||||
json={"model": model_name, "input": [input_tokens] * N},
|
||||
)
|
||||
output = ClassificationResponse.model_validate(classification_response.json())
|
||||
|
||||
assert len(output.data) == N
|
||||
for i, item in enumerate(output.data):
|
||||
assert item.index == i
|
||||
assert hasattr(item, "label")
|
||||
assert hasattr(item, "probs")
|
||||
assert len(item.probs) == item.num_classes
|
||||
assert item.label in ["Default", "Spoiled"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
def test_empty_input_error(server: RemoteOpenAIServer, model_name: str):
|
||||
classification_response = requests.post(
|
||||
server.url_for("classify"),
|
||||
json={"model": model_name, "input": ""},
|
||||
)
|
||||
|
||||
error = classification_response.json()
|
||||
assert classification_response.status_code == 400
|
||||
assert "error" in error
|
||||
|
||||
classification_response = requests.post(
|
||||
server.url_for("classify"),
|
||||
json={"model": model_name, "input": []},
|
||||
)
|
||||
|
||||
error = classification_response.json()
|
||||
assert classification_response.status_code == 400
|
||||
assert "error" in error
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
def test_truncate_prompt_tokens(server: RemoteOpenAIServer, model_name: str):
|
||||
long_text = "hello " * 600
|
||||
|
||||
classification_response = requests.post(
|
||||
server.url_for("classify"),
|
||||
json={"model": model_name, "input": long_text, "truncate_prompt_tokens": 5},
|
||||
)
|
||||
|
||||
classification_response.raise_for_status()
|
||||
output = ClassificationResponse.model_validate(classification_response.json())
|
||||
|
||||
assert len(output.data) == 1
|
||||
assert output.data[0].index == 0
|
||||
assert hasattr(output.data[0], "probs")
|
||||
assert output.usage.prompt_tokens == 5
|
||||
assert output.usage.total_tokens == 5
|
||||
|
||||
# invalid_truncate_prompt_tokens
|
||||
classification_response = requests.post(
|
||||
server.url_for("classify"),
|
||||
json={"model": model_name, "input": "test", "truncate_prompt_tokens": 513},
|
||||
)
|
||||
|
||||
error = classification_response.json()
|
||||
assert classification_response.status_code == 400
|
||||
assert "truncate_prompt_tokens" in error["error"]["message"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
def test_add_special_tokens(server: RemoteOpenAIServer, model_name: str):
|
||||
# The add_special_tokens parameter doesn't seem to be working with this model.
|
||||
# working with papluca/xlm-roberta-base-language-detection
|
||||
response = requests.post(
|
||||
server.url_for("classify"),
|
||||
json={"model": model_name, "input": input_text, "add_special_tokens": False},
|
||||
)
|
||||
response.raise_for_status()
|
||||
ClassificationResponse.model_validate(response.json())
|
||||
|
||||
response = requests.post(
|
||||
server.url_for("classify"),
|
||||
json={"model": model_name, "input": input_text, "add_special_tokens": True},
|
||||
)
|
||||
response.raise_for_status()
|
||||
ClassificationResponse.model_validate(response.json())
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
async def test_chat_request(server: RemoteOpenAIServer, model_name: str):
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "The cat sat on the mat.",
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "A feline was resting on a rug.",
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Stars twinkle brightly in the night sky.",
|
||||
},
|
||||
]
|
||||
|
||||
# test chat request basic usage
|
||||
response = requests.post(
|
||||
server.url_for("classify"),
|
||||
json={"model": model_name, "messages": messages},
|
||||
)
|
||||
|
||||
response.raise_for_status()
|
||||
output = ClassificationResponse.model_validate(response.json())
|
||||
|
||||
assert output.object == "list"
|
||||
assert output.model == MODEL_NAME
|
||||
assert len(output.data) == 1
|
||||
assert hasattr(output.data[0], "label")
|
||||
assert hasattr(output.data[0], "probs")
|
||||
assert output.usage.prompt_tokens == 51
|
||||
|
||||
# test add_generation_prompt
|
||||
response = requests.post(
|
||||
server.url_for("classify"),
|
||||
json={"model": model_name, "messages": messages, "add_generation_prompt": True},
|
||||
)
|
||||
|
||||
response.raise_for_status()
|
||||
output = ClassificationResponse.model_validate(response.json())
|
||||
|
||||
assert output.object == "list"
|
||||
assert output.model == MODEL_NAME
|
||||
assert len(output.data) == 1
|
||||
assert hasattr(output.data[0], "label")
|
||||
assert hasattr(output.data[0], "probs")
|
||||
assert output.usage.prompt_tokens == 54
|
||||
|
||||
# test continue_final_message
|
||||
response = requests.post(
|
||||
server.url_for("classify"),
|
||||
json={
|
||||
"model": model_name,
|
||||
"messages": messages,
|
||||
"continue_final_message": True,
|
||||
},
|
||||
)
|
||||
|
||||
response.raise_for_status()
|
||||
output = ClassificationResponse.model_validate(response.json())
|
||||
|
||||
assert output.object == "list"
|
||||
assert output.model == MODEL_NAME
|
||||
assert len(output.data) == 1
|
||||
assert hasattr(output.data[0], "label")
|
||||
assert hasattr(output.data[0], "probs")
|
||||
assert output.usage.prompt_tokens == 49
|
||||
|
||||
# test add_special_tokens
|
||||
# The add_special_tokens parameter doesn't seem to be working with this model.
|
||||
response = requests.post(
|
||||
server.url_for("classify"),
|
||||
json={"model": model_name, "messages": messages, "add_special_tokens": True},
|
||||
)
|
||||
|
||||
response.raise_for_status()
|
||||
output = ClassificationResponse.model_validate(response.json())
|
||||
|
||||
assert output.object == "list"
|
||||
assert output.model == MODEL_NAME
|
||||
assert len(output.data) == 1
|
||||
assert hasattr(output.data[0], "label")
|
||||
assert hasattr(output.data[0], "probs")
|
||||
assert output.usage.prompt_tokens == 51
|
||||
|
||||
# test continue_final_message with add_generation_prompt
|
||||
response = requests.post(
|
||||
server.url_for("classify"),
|
||||
json={
|
||||
"model": model_name,
|
||||
"messages": messages,
|
||||
"continue_final_message": True,
|
||||
"add_generation_prompt": True,
|
||||
},
|
||||
)
|
||||
assert (
|
||||
"Cannot set both `continue_final_message` and `add_generation_prompt` to True."
|
||||
in response.json()["error"]["message"]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invocations_completion_request(server: RemoteOpenAIServer):
|
||||
request_args = {
|
||||
"model": MODEL_NAME,
|
||||
"input": input_text,
|
||||
}
|
||||
|
||||
classification_response = requests.post(
|
||||
server.url_for("classify"), json=request_args
|
||||
)
|
||||
classification_response.raise_for_status()
|
||||
|
||||
invocation_response = requests.post(
|
||||
server.url_for("invocations"), json=request_args
|
||||
)
|
||||
invocation_response.raise_for_status()
|
||||
|
||||
classification_output = classification_response.json()
|
||||
invocation_output = invocation_response.json()
|
||||
|
||||
assert classification_output.keys() == invocation_output.keys()
|
||||
for classification_data, invocation_data in zip(
|
||||
classification_output["data"], invocation_output["data"]
|
||||
):
|
||||
assert classification_data.keys() == invocation_data.keys()
|
||||
assert classification_data["probs"] == pytest.approx(
|
||||
invocation_data["probs"], rel=0.01
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invocations_chat_request(server: RemoteOpenAIServer):
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "The cat sat on the mat.",
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "A feline was resting on a rug.",
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Stars twinkle brightly in the night sky.",
|
||||
},
|
||||
]
|
||||
|
||||
request_args = {"model": MODEL_NAME, "messages": messages}
|
||||
|
||||
classification_response = requests.post(
|
||||
server.url_for("classify"), json=request_args
|
||||
)
|
||||
classification_response.raise_for_status()
|
||||
|
||||
invocation_response = requests.post(
|
||||
server.url_for("invocations"), json=request_args
|
||||
)
|
||||
invocation_response.raise_for_status()
|
||||
|
||||
classification_output = classification_response.json()
|
||||
invocation_output = invocation_response.json()
|
||||
|
||||
assert classification_output.keys() == invocation_output.keys()
|
||||
for classification_data, invocation_data in zip(
|
||||
classification_output["data"], invocation_output["data"]
|
||||
):
|
||||
assert classification_data.keys() == invocation_data.keys()
|
||||
assert classification_data["probs"] == pytest.approx(
|
||||
invocation_data["probs"], rel=0.01
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
async def test_use_activation(server: RemoteOpenAIServer, model_name: str):
|
||||
async def get_outputs(use_activation):
|
||||
response = requests.post(
|
||||
server.url_for("classify"),
|
||||
json={
|
||||
"model": model_name,
|
||||
"input": input_text,
|
||||
"use_activation": use_activation,
|
||||
},
|
||||
)
|
||||
outputs = response.json()
|
||||
return torch.tensor([x["probs"] for x in outputs["data"]])
|
||||
|
||||
default = await get_outputs(use_activation=None)
|
||||
w_activation = await get_outputs(use_activation=True)
|
||||
wo_activation = await get_outputs(use_activation=False)
|
||||
|
||||
assert torch.allclose(default, w_activation, atol=1e-2), (
|
||||
"Default should use activation."
|
||||
)
|
||||
assert not torch.allclose(w_activation, wo_activation, atol=1e-2), (
|
||||
"wo_activation should not use activation."
|
||||
)
|
||||
assert torch.allclose(F.softmax(wo_activation, dim=-1), w_activation, atol=1e-2), (
|
||||
"w_activation should be close to activation(wo_activation)."
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
async def test_score(server: RemoteOpenAIServer, model_name: str):
|
||||
# Scoring API is only enabled for num_labels == 1.
|
||||
response = requests.post(
|
||||
server.url_for("score"),
|
||||
json={
|
||||
"model": model_name,
|
||||
"queries": "ping",
|
||||
"documents": "pong",
|
||||
},
|
||||
)
|
||||
assert response.json()["detail"] == "Not Found"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
async def test_rerank(server: RemoteOpenAIServer, model_name: str):
|
||||
# Scoring API is only enabled for num_labels == 1.
|
||||
response = requests.post(
|
||||
server.url_for("rerank"),
|
||||
json={
|
||||
"model": model_name,
|
||||
"query": "ping",
|
||||
"documents": ["pong"],
|
||||
},
|
||||
)
|
||||
assert response.json()["detail"] == "Not Found"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
async def test_pooling_classify(server: RemoteOpenAIServer, model_name: str):
|
||||
response = requests.post(
|
||||
server.url_for("pooling"),
|
||||
json={
|
||||
"model": model_name,
|
||||
"input": input_text,
|
||||
"encoding_format": "float",
|
||||
"task": "classify",
|
||||
},
|
||||
)
|
||||
poolings = PoolingResponse.model_validate(response.json())
|
||||
assert len(poolings.data) == 1
|
||||
assert len(poolings.data[0].data) == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
@pytest.mark.parametrize("task", ["embed", "token_embed", "token_classify", "plugin"])
|
||||
async def test_pooling_not_supported(
|
||||
server: RemoteOpenAIServer, model_name: str, task: str
|
||||
):
|
||||
response = requests.post(
|
||||
server.url_for("pooling"),
|
||||
json={
|
||||
"model": model_name,
|
||||
"input": input_text,
|
||||
"encoding_format": "float",
|
||||
"task": task,
|
||||
},
|
||||
)
|
||||
assert response.json()["error"]["type"] == "BadRequestError"
|
||||
|
||||
if task == "plugin":
|
||||
err_msg = "No IOProcessor plugin installed."
|
||||
elif task == "token_classify":
|
||||
err_msg = "Try switching the model's pooling_task via"
|
||||
else:
|
||||
err_msg = f"Unsupported task: {task!r}"
|
||||
assert response.json()["error"]["message"].startswith(err_msg)
|
||||
@@ -1,147 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import json
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.assets.base import VLLM_S3_BUCKET_URL
|
||||
from vllm.entrypoints.pooling.classify.protocol import ClassificationResponse
|
||||
from vllm.multimodal.utils import encode_image_url, fetch_image
|
||||
|
||||
MODEL_NAME = "muziyongshixin/Qwen2.5-VL-7B-for-VideoCls"
|
||||
MAXIMUM_VIDEOS = 1
|
||||
|
||||
HF_OVERRIDES = {"architectures": ["Qwen2_5_VLForSequenceClassification"]}
|
||||
input_text = "This product was excellent and exceeded my expectations"
|
||||
image_url = f"{VLLM_S3_BUCKET_URL}/multimodal_asset/cat_snow.jpg"
|
||||
image_base64 = {"url": encode_image_url(fetch_image(image_url))}
|
||||
video_url = f"{VLLM_S3_BUCKET_URL}/multimodal_asset/slow_traffic_small.mp4"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
args = [
|
||||
"--runner",
|
||||
"pooling",
|
||||
"--max-model-len",
|
||||
"16384",
|
||||
"--enforce-eager",
|
||||
"--limit-mm-per-prompt",
|
||||
json.dumps({"video": MAXIMUM_VIDEOS}),
|
||||
]
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
MODEL_NAME, args, override_hf_configs=HF_OVERRIDES
|
||||
) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
def test_chat_text_request(server: RemoteOpenAIServer, model_name: str):
|
||||
messages = [
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "Please classify this text request.",
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": input_text,
|
||||
},
|
||||
]
|
||||
|
||||
response = requests.post(
|
||||
server.url_for("classify"),
|
||||
json={"model": model_name, "messages": messages},
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
output = ClassificationResponse.model_validate(response.json())
|
||||
|
||||
assert output.object == "list"
|
||||
assert output.model == model_name
|
||||
assert len(output.data) == 1
|
||||
assert len(output.data[0].probs) == 2
|
||||
assert output.usage.prompt_tokens == 35
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
def test_chat_image_url_request(server: RemoteOpenAIServer, model_name: str):
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Please classify this image."},
|
||||
{"type": "image_url", "image_url": {"url": image_url}},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
response = requests.post(
|
||||
server.url_for("classify"),
|
||||
json={"model": model_name, "messages": messages},
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
output = ClassificationResponse.model_validate(response.json())
|
||||
|
||||
assert output.object == "list"
|
||||
assert output.model == model_name
|
||||
assert len(output.data) == 1
|
||||
assert len(output.data[0].probs) == 2
|
||||
assert output.usage.prompt_tokens == 47
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
def test_chat_image_base64_request(server: RemoteOpenAIServer, model_name: str):
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Please classify this image."},
|
||||
{"type": "image_url", "image_url": image_base64},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
response = requests.post(
|
||||
server.url_for("classify"),
|
||||
json={"model": model_name, "messages": messages},
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
output = ClassificationResponse.model_validate(response.json())
|
||||
|
||||
assert output.object == "list"
|
||||
assert output.model == model_name
|
||||
assert len(output.data) == 1
|
||||
assert len(output.data[0].probs) == 2
|
||||
assert output.usage.prompt_tokens == 47
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
def test_chat_video_url_request(server: RemoteOpenAIServer, model_name: str):
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Please classify this video."},
|
||||
{"type": "video_url", "video_url": {"url": video_url}},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
response = requests.post(
|
||||
server.url_for("classify"),
|
||||
json={"model": model_name, "messages": messages},
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
output = ClassificationResponse.model_validate(response.json())
|
||||
|
||||
assert output.object == "list"
|
||||
assert output.model == model_name
|
||||
assert len(output.data) == 1
|
||||
assert len(output.data[0].probs) == 2
|
||||
assert output.usage.prompt_tokens == 8993
|
||||
@@ -1,310 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the Cohere /v2/embed API with generic (non-Cohere) models.
|
||||
|
||||
Validates that the Cohere v2 embed endpoint works correctly with standard
|
||||
embedding models, covering text embedding, embedding type conversions,
|
||||
response structure, batching, normalisation, and semantic similarity.
|
||||
"""
|
||||
|
||||
import struct
|
||||
|
||||
import numpy as np
|
||||
import pybase64 as base64
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
|
||||
DTYPE = "bfloat16"
|
||||
|
||||
MODELS: list[tuple[str, list[str]]] = [
|
||||
("intfloat/multilingual-e5-small", []),
|
||||
(
|
||||
"Snowflake/snowflake-arctic-embed-m-v1.5",
|
||||
[
|
||||
"--trust_remote_code",
|
||||
"--hf_overrides",
|
||||
'{"matryoshka_dimensions":[256]}',
|
||||
],
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", params=MODELS, ids=lambda m: m[0])
|
||||
def model_config(request):
|
||||
return request.param
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def model_name(model_config):
|
||||
return model_config[0]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server(model_config):
|
||||
name, extra_args = model_config
|
||||
args = [
|
||||
"--runner",
|
||||
"pooling",
|
||||
"--dtype",
|
||||
DTYPE,
|
||||
"--enforce-eager",
|
||||
"--max-model-len",
|
||||
"512",
|
||||
"--gpu-memory-utilization",
|
||||
"0.02",
|
||||
] + extra_args
|
||||
with RemoteOpenAIServer(name, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
def _cohere_embed(
|
||||
server: RemoteOpenAIServer,
|
||||
model_name: str,
|
||||
texts: list[str] | None = None,
|
||||
images: list[str] | None = None,
|
||||
input_type: str | None = None,
|
||||
embedding_types: list[str] | None = None,
|
||||
) -> dict:
|
||||
body: dict = {"model": model_name}
|
||||
if input_type is not None:
|
||||
body["input_type"] = input_type
|
||||
if texts is not None:
|
||||
body["texts"] = texts
|
||||
if images is not None:
|
||||
body["images"] = images
|
||||
if embedding_types is not None:
|
||||
body["embedding_types"] = embedding_types
|
||||
resp = requests.post(server.url_for("/v2/embed"), json=body)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
|
||||
def _openai_embed(
|
||||
server: RemoteOpenAIServer, model_name: str, texts: list[str]
|
||||
) -> dict:
|
||||
body = {"model": model_name, "input": texts, "encoding_format": "float"}
|
||||
resp = requests.post(server.url_for("/v1/embeddings"), json=body)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
|
||||
def _cosine_sim(a: list[float], b: list[float]) -> float:
|
||||
va, vb = np.array(a), np.array(b)
|
||||
return float(np.dot(va, vb) / (np.linalg.norm(va) * np.linalg.norm(vb)))
|
||||
|
||||
|
||||
# -----------------------------------------------------------
|
||||
# Text embedding tests
|
||||
# -----------------------------------------------------------
|
||||
|
||||
|
||||
def test_basic_embed(server: RemoteOpenAIServer, model_name: str):
|
||||
r = _cohere_embed(
|
||||
server, model_name, texts=["hello world"], embedding_types=["float"]
|
||||
)
|
||||
assert "embeddings" in r
|
||||
assert len(r["embeddings"]["float"]) == 1
|
||||
assert len(r["embeddings"]["float"][0]) > 0
|
||||
|
||||
|
||||
def test_unsupported_input_type_rejected(server: RemoteOpenAIServer, model_name: str):
|
||||
"""An input_type not defined in the model's prompt config should be
|
||||
rejected with a 400 error."""
|
||||
body = {
|
||||
"model": model_name,
|
||||
"input_type": "nonexistent_type",
|
||||
"texts": ["hello world"],
|
||||
"embedding_types": ["float"],
|
||||
}
|
||||
resp = requests.post(server.url_for("/v2/embed"), json=body)
|
||||
assert resp.status_code == 400
|
||||
assert "Unsupported input_type" in resp.json()["error"]["message"]
|
||||
|
||||
|
||||
def test_omitted_input_type_accepted(server: RemoteOpenAIServer, model_name: str):
|
||||
"""Omitting input_type should always work (no prompt prefix applied)."""
|
||||
body = {
|
||||
"model": model_name,
|
||||
"texts": ["hello world"],
|
||||
"embedding_types": ["float"],
|
||||
}
|
||||
resp = requests.post(server.url_for("/v2/embed"), json=body)
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert len(data["embeddings"]["float"]) == 1
|
||||
|
||||
|
||||
def test_v1_v2_parity(server: RemoteOpenAIServer, model_name: str):
|
||||
"""v1 (OpenAI) and v2 (Cohere) endpoints should produce the same
|
||||
float embeddings for a generic model."""
|
||||
texts = ["hello world"]
|
||||
v2 = _cohere_embed(server, model_name, texts=texts, embedding_types=["float"])
|
||||
v1 = _openai_embed(server, model_name, texts)
|
||||
cos = _cosine_sim(v2["embeddings"]["float"][0], v1["data"][0]["embedding"])
|
||||
assert cos > 0.9999, f"v1/v2 parity failed, cosine={cos}"
|
||||
|
||||
|
||||
def test_embedding_types(server: RemoteOpenAIServer, model_name: str):
|
||||
r = _cohere_embed(
|
||||
server,
|
||||
model_name,
|
||||
texts=["test"],
|
||||
embedding_types=["float", "binary", "ubinary"],
|
||||
)
|
||||
dim = len(r["embeddings"]["float"][0])
|
||||
assert len(r["embeddings"]["binary"][0]) == dim // 8
|
||||
assert len(r["embeddings"]["ubinary"][0]) == dim // 8
|
||||
|
||||
|
||||
def test_response_structure(server: RemoteOpenAIServer, model_name: str):
|
||||
r = _cohere_embed(server, model_name, texts=["test"], embedding_types=["float"])
|
||||
assert "id" in r
|
||||
assert "embeddings" in r
|
||||
assert "texts" in r
|
||||
assert r["texts"] == ["test"]
|
||||
assert "meta" in r
|
||||
assert r["meta"]["api_version"]["version"] == "2"
|
||||
assert "billed_units" in r["meta"]
|
||||
assert r["meta"]["billed_units"]["input_tokens"] > 0
|
||||
assert r["meta"]["billed_units"]["image_tokens"] == 0
|
||||
|
||||
|
||||
def test_batch(server: RemoteOpenAIServer, model_name: str):
|
||||
texts = ["apple", "banana", "cherry"]
|
||||
r = _cohere_embed(server, model_name, texts=texts, embedding_types=["float"])
|
||||
assert len(r["embeddings"]["float"]) == 3
|
||||
dim = len(r["embeddings"]["float"][0])
|
||||
for emb in r["embeddings"]["float"]:
|
||||
assert len(emb) == dim
|
||||
|
||||
|
||||
def test_l2_normalized(server: RemoteOpenAIServer, model_name: str):
|
||||
r = _cohere_embed(
|
||||
server, model_name, texts=["hello world"], embedding_types=["float"]
|
||||
)
|
||||
emb = np.array(r["embeddings"]["float"][0])
|
||||
assert abs(float(np.linalg.norm(emb)) - 1.0) < 0.01
|
||||
|
||||
|
||||
def test_semantic_similarity(server: RemoteOpenAIServer, model_name: str):
|
||||
r = _cohere_embed(
|
||||
server,
|
||||
model_name,
|
||||
texts=["machine learning", "deep learning", "chocolate cake recipe"],
|
||||
embedding_types=["float"],
|
||||
)
|
||||
embs = r["embeddings"]["float"]
|
||||
cos_related = _cosine_sim(embs[0], embs[1])
|
||||
cos_unrelated = _cosine_sim(embs[0], embs[2])
|
||||
assert cos_related > cos_unrelated
|
||||
|
||||
|
||||
def test_missing_input_returns_error(server: RemoteOpenAIServer, model_name: str):
|
||||
body = {"model": model_name}
|
||||
resp = requests.post(server.url_for("/v2/embed"), json=body)
|
||||
assert resp.status_code == 400
|
||||
|
||||
|
||||
def test_base64_embedding_type(server: RemoteOpenAIServer, model_name: str):
|
||||
r = _cohere_embed(
|
||||
server,
|
||||
model_name,
|
||||
texts=["test encoding"],
|
||||
embedding_types=["float", "base64"],
|
||||
)
|
||||
float_emb = r["embeddings"]["float"][0]
|
||||
b64_str = r["embeddings"]["base64"][0]
|
||||
decoded = struct.unpack(f"<{len(float_emb)}f", base64.b64decode(b64_str))
|
||||
np.testing.assert_allclose(float_emb, decoded, rtol=1e-5)
|
||||
|
||||
|
||||
# -----------------------------------------------------------
|
||||
# Truncation tests
|
||||
# -----------------------------------------------------------
|
||||
|
||||
|
||||
def _cohere_embed_raw(
|
||||
server: RemoteOpenAIServer,
|
||||
body: dict,
|
||||
) -> requests.Response:
|
||||
return requests.post(server.url_for("/v2/embed"), json=body)
|
||||
|
||||
|
||||
def test_truncate_end_succeeds(server: RemoteOpenAIServer, model_name: str):
|
||||
"""truncate=END should silently truncate long input."""
|
||||
long_text = " ".join(["word"] * 2000)
|
||||
body = {
|
||||
"model": model_name,
|
||||
"texts": [long_text],
|
||||
"embedding_types": ["float"],
|
||||
"truncate": "END",
|
||||
}
|
||||
resp = _cohere_embed_raw(server, body)
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert len(data["embeddings"]["float"]) == 1
|
||||
|
||||
|
||||
def test_truncate_start_succeeds(server: RemoteOpenAIServer, model_name: str):
|
||||
"""truncate=START should silently truncate long input from the start."""
|
||||
long_text = " ".join(["word"] * 2000)
|
||||
body = {
|
||||
"model": model_name,
|
||||
"texts": [long_text],
|
||||
"embedding_types": ["float"],
|
||||
"truncate": "START",
|
||||
}
|
||||
resp = _cohere_embed_raw(server, body)
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert len(data["embeddings"]["float"]) == 1
|
||||
|
||||
|
||||
def test_truncate_none_rejects_long_input(server: RemoteOpenAIServer, model_name: str):
|
||||
"""truncate=NONE should error when input exceeds model context."""
|
||||
long_text = " ".join(["word"] * 2000)
|
||||
body = {
|
||||
"model": model_name,
|
||||
"texts": [long_text],
|
||||
"embedding_types": ["float"],
|
||||
"truncate": "NONE",
|
||||
}
|
||||
resp = _cohere_embed_raw(server, body)
|
||||
assert resp.status_code == 400
|
||||
|
||||
|
||||
def test_truncate_start_vs_end_differ(server: RemoteOpenAIServer, model_name: str):
|
||||
"""START and END truncation should produce different embeddings
|
||||
when the input is long enough to actually be truncated.
|
||||
|
||||
We construct input with distinct tokens at the start vs end
|
||||
so that keeping different halves produces different embeddings.
|
||||
"""
|
||||
start_words = " ".join([f"alpha{i}" for i in range(300)])
|
||||
end_words = " ".join([f"omega{i}" for i in range(300)])
|
||||
long_text = start_words + " " + end_words
|
||||
|
||||
body_end = {
|
||||
"model": model_name,
|
||||
"texts": [long_text],
|
||||
"embedding_types": ["float"],
|
||||
"truncate": "END",
|
||||
}
|
||||
body_start = {
|
||||
"model": model_name,
|
||||
"texts": [long_text],
|
||||
"embedding_types": ["float"],
|
||||
"truncate": "START",
|
||||
}
|
||||
r_end = _cohere_embed_raw(server, body_end).json()
|
||||
r_start = _cohere_embed_raw(server, body_start).json()
|
||||
|
||||
emb_end = r_end["embeddings"]["float"][0]
|
||||
emb_start = r_start["embeddings"]["float"][0]
|
||||
cos = _cosine_sim(emb_end, emb_start)
|
||||
assert cos < 0.99, (
|
||||
f"START and END truncation should produce different embeddings "
|
||||
f"for long input, but cosine similarity was {cos}"
|
||||
)
|
||||
@@ -1,135 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the Cohere /v2/embed API with a multimodal model (SigLIP).
|
||||
|
||||
Validates image embedding, batching, normalisation, and embedding type
|
||||
conversions through the /v2/embed endpoint.
|
||||
"""
|
||||
|
||||
import struct
|
||||
import zlib
|
||||
|
||||
import numpy as np
|
||||
import pybase64 as base64
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
|
||||
MODEL_NAME = "google/siglip-so400m-patch14-384"
|
||||
DTYPE = "bfloat16"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
args = [
|
||||
"--runner",
|
||||
"pooling",
|
||||
"--dtype",
|
||||
DTYPE,
|
||||
"--enforce-eager",
|
||||
"--max-model-len",
|
||||
"64",
|
||||
"--gpu-memory-utilization",
|
||||
"0.3",
|
||||
]
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
def _make_tiny_png(r: int, g: int, b: int, w: int = 2, h: int = 2) -> str:
|
||||
raw = b""
|
||||
for _ in range(h):
|
||||
raw += b"\x00" + bytes([r, g, b]) * w
|
||||
compressed = zlib.compress(raw)
|
||||
|
||||
def chunk(ctype: bytes, cdata: bytes) -> bytes:
|
||||
c = ctype + cdata
|
||||
return (
|
||||
struct.pack(">I", len(cdata))
|
||||
+ c
|
||||
+ struct.pack(">I", zlib.crc32(c) & 0xFFFFFFFF)
|
||||
)
|
||||
|
||||
ihdr = struct.pack(">IIBBBBB", w, h, 8, 2, 0, 0, 0)
|
||||
png = (
|
||||
b"\x89PNG\r\n\x1a\n"
|
||||
+ chunk(b"IHDR", ihdr)
|
||||
+ chunk(b"IDAT", compressed)
|
||||
+ chunk(b"IEND", b"")
|
||||
)
|
||||
return "data:image/png;base64," + base64.b64encode(png).decode()
|
||||
|
||||
|
||||
def _cohere_embed(
|
||||
server: RemoteOpenAIServer,
|
||||
texts: list[str] | None = None,
|
||||
images: list[str] | None = None,
|
||||
embedding_types: list[str] | None = None,
|
||||
) -> dict:
|
||||
body: dict = {"model": MODEL_NAME}
|
||||
if texts is not None:
|
||||
body["texts"] = texts
|
||||
if images is not None:
|
||||
body["images"] = images
|
||||
if embedding_types is not None:
|
||||
body["embedding_types"] = embedding_types
|
||||
resp = requests.post(server.url_for("/v2/embed"), json=body)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
|
||||
def test_image_embed(server: RemoteOpenAIServer):
|
||||
img_uri = _make_tiny_png(255, 0, 0)
|
||||
r = _cohere_embed(
|
||||
server,
|
||||
images=[img_uri],
|
||||
embedding_types=["float"],
|
||||
)
|
||||
assert "embeddings" in r
|
||||
assert len(r["embeddings"]["float"]) == 1
|
||||
assert len(r["embeddings"]["float"][0]) > 0
|
||||
assert r["meta"]["billed_units"]["image_tokens"] > 0
|
||||
assert r["meta"]["billed_units"]["input_tokens"] == 0
|
||||
|
||||
|
||||
def test_image_batch(server: RemoteOpenAIServer):
|
||||
red = _make_tiny_png(255, 0, 0)
|
||||
blue = _make_tiny_png(0, 0, 255)
|
||||
r = _cohere_embed(
|
||||
server,
|
||||
images=[red, blue],
|
||||
embedding_types=["float"],
|
||||
)
|
||||
assert len(r["embeddings"]["float"]) == 2
|
||||
|
||||
|
||||
def test_image_l2_normalized(server: RemoteOpenAIServer):
|
||||
img_uri = _make_tiny_png(0, 255, 0)
|
||||
r = _cohere_embed(
|
||||
server,
|
||||
images=[img_uri],
|
||||
embedding_types=["float"],
|
||||
)
|
||||
emb = np.array(r["embeddings"]["float"][0])
|
||||
assert abs(float(np.linalg.norm(emb)) - 1.0) < 0.01
|
||||
|
||||
|
||||
def test_image_embedding_types(server: RemoteOpenAIServer):
|
||||
img_uri = _make_tiny_png(128, 128, 128)
|
||||
r = _cohere_embed(
|
||||
server,
|
||||
images=[img_uri],
|
||||
embedding_types=["float", "binary", "ubinary"],
|
||||
)
|
||||
dim = len(r["embeddings"]["float"][0])
|
||||
assert len(r["embeddings"]["binary"][0]) == dim // 8
|
||||
assert len(r["embeddings"]["ubinary"][0]) == dim // 8
|
||||
|
||||
|
||||
def test_text_embed_on_multimodal(server: RemoteOpenAIServer):
|
||||
"""SigLIP also supports text-only embedding via /v2/embed."""
|
||||
r = _cohere_embed(server, texts=["hello world"], embedding_types=["float"])
|
||||
assert "embeddings" in r
|
||||
assert len(r["embeddings"]["float"]) == 1
|
||||
assert len(r["embeddings"]["float"][0]) > 0
|
||||
@@ -1,121 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Parity test between Cohere /v2/embed and OpenAI /v1/embeddings.
|
||||
|
||||
Verifies that both endpoints produce identical float embeddings when
|
||||
no prompt prefix is applied (input_type omitted for Cohere /v2/embed).
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from tests.utils import ROCM_EXTRA_ARGS, RemoteOpenAIServer
|
||||
|
||||
MODEL_NAME = "BAAI/bge-base-en-v1.5"
|
||||
DTYPE = "bfloat16"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
args = [
|
||||
"--runner",
|
||||
"pooling",
|
||||
"--dtype",
|
||||
DTYPE,
|
||||
"--enforce-eager",
|
||||
"--max-model-len",
|
||||
"512",
|
||||
"--gpu-memory-utilization",
|
||||
"0.02",
|
||||
] + ROCM_EXTRA_ARGS
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
def _cohere_embed(
|
||||
server: RemoteOpenAIServer,
|
||||
texts: list[str],
|
||||
) -> list[list[float]]:
|
||||
body = {
|
||||
"model": MODEL_NAME,
|
||||
"texts": texts,
|
||||
"embedding_types": ["float"],
|
||||
}
|
||||
resp = requests.post(server.url_for("/v2/embed"), json=body)
|
||||
resp.raise_for_status()
|
||||
return resp.json()["embeddings"]["float"]
|
||||
|
||||
|
||||
def _openai_embed(
|
||||
server: RemoteOpenAIServer,
|
||||
texts: list[str],
|
||||
) -> list[list[float]]:
|
||||
body = {"model": MODEL_NAME, "input": texts, "encoding_format": "float"}
|
||||
resp = requests.post(server.url_for("/v1/embeddings"), json=body)
|
||||
resp.raise_for_status()
|
||||
return [item["embedding"] for item in resp.json()["data"]]
|
||||
|
||||
|
||||
def _cosine_sim(a: list[float], b: list[float]) -> float:
|
||||
va, vb = np.array(a), np.array(b)
|
||||
return float(np.dot(va, vb) / (np.linalg.norm(va) * np.linalg.norm(vb)))
|
||||
|
||||
|
||||
def test_single_text_parity(server: RemoteOpenAIServer):
|
||||
"""A single text should produce equivalent embeddings via both APIs."""
|
||||
texts = ["the quick brown fox jumps over the lazy dog"]
|
||||
v2 = _cohere_embed(server, texts)
|
||||
v1 = _openai_embed(server, texts)
|
||||
# Full-suite BF16 runs can introduce tiny numerical drift even when both
|
||||
# endpoints are functionally equivalent, so compare semantic equivalence
|
||||
# instead of exact elementwise equality.
|
||||
cos = _cosine_sim(v2[0], v1[0])
|
||||
assert cos > 0.9999, f"single-text parity failed, cosine={cos}"
|
||||
|
||||
|
||||
def test_batch_parity(server: RemoteOpenAIServer):
|
||||
"""A batch of texts should produce equivalent embeddings via both APIs,
|
||||
in the same order."""
|
||||
texts = [
|
||||
"machine learning",
|
||||
"deep learning",
|
||||
"natural language processing",
|
||||
]
|
||||
v2 = _cohere_embed(server, texts)
|
||||
v1 = _openai_embed(server, texts)
|
||||
assert len(v2) == len(v1) == 3
|
||||
|
||||
similarities = np.array(
|
||||
[[_cosine_sim(v2_emb, v1_emb) for v1_emb in v1] for v2_emb in v2]
|
||||
)
|
||||
for i in range(3):
|
||||
assert int(np.argmax(similarities[i])) == i, (
|
||||
f"batch parity order mismatch at index {i}: "
|
||||
f"similarities={similarities[i].tolist()}"
|
||||
)
|
||||
assert similarities[i, i] > 0.9999, (
|
||||
f"batch parity failed at index {i}, cosine={similarities[i, i]}"
|
||||
)
|
||||
|
||||
|
||||
def test_token_count_parity(server: RemoteOpenAIServer):
|
||||
"""Both APIs should report the same prompt token count."""
|
||||
texts = ["hello world"]
|
||||
v2_resp = requests.post(
|
||||
server.url_for("/v2/embed"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"texts": texts,
|
||||
"embedding_types": ["float"],
|
||||
},
|
||||
)
|
||||
v1_resp = requests.post(
|
||||
server.url_for("/v1/embeddings"),
|
||||
json={"model": MODEL_NAME, "input": texts, "encoding_format": "float"},
|
||||
)
|
||||
v2_resp.raise_for_status()
|
||||
v1_resp.raise_for_status()
|
||||
v2_tokens = v2_resp.json()["meta"]["billed_units"]["input_tokens"]
|
||||
v1_tokens = v1_resp.json()["usage"]["prompt_tokens"]
|
||||
assert v2_tokens == v1_tokens
|
||||
@@ -1,41 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.models.language.pooling_mteb_test.mteb_embed_utils import (
|
||||
MTEB_EMBED_TASKS,
|
||||
MTEB_EMBED_TOL,
|
||||
OpenAIClientMtebEncoder,
|
||||
run_mteb_embed_task,
|
||||
)
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
|
||||
os.environ["VLLM_LOGGING_LEVEL"] = "WARNING"
|
||||
|
||||
MODEL_NAME = "intfloat/e5-small"
|
||||
MAIN_SCORE = 0.7422994752439667
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
args = ["--runner", "pooling", "--enforce-eager", "--disable-uvicorn-access-log"]
|
||||
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
def test_mteb_embed(server):
|
||||
client = server.get_client()
|
||||
encoder = OpenAIClientMtebEncoder(MODEL_NAME, client)
|
||||
vllm_main_score = run_mteb_embed_task(encoder, MTEB_EMBED_TASKS)
|
||||
st_main_score = MAIN_SCORE
|
||||
|
||||
print("VLLM main score: ", vllm_main_score)
|
||||
print("SentenceTransformer main score: ", st_main_score)
|
||||
print("Difference: ", st_main_score - vllm_main_score)
|
||||
|
||||
# We are not concerned that the vllm mteb results are better
|
||||
# than SentenceTransformers, so we only perform one-sided testing.
|
||||
assert st_main_score - vllm_main_score < MTEB_EMBED_TOL
|
||||
@@ -1,507 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for EmbedIOProcessor."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
from vllm import PoolingParams
|
||||
from vllm.entrypoints.pooling.embed.io_processor import EmbedIOProcessor
|
||||
from vllm.entrypoints.pooling.embed.protocol import (
|
||||
CohereEmbedRequest,
|
||||
EmbeddingBatchChatInputRequest,
|
||||
EmbeddingBatchChatRequest,
|
||||
EmbeddingChatInputRequest,
|
||||
EmbeddingChatRequest,
|
||||
EmbeddingCompletionRequest,
|
||||
EmbeddingRequest,
|
||||
)
|
||||
from vllm.entrypoints.pooling.typing import (
|
||||
PoolingEngineInput,
|
||||
PoolingServeContext,
|
||||
)
|
||||
from vllm.inputs import tokens_input
|
||||
from vllm.outputs import PoolingOutput, PoolingRequestOutput
|
||||
|
||||
|
||||
class TestEmbeddingRequestParsing:
|
||||
"""Unit tests for OpenAI embedding request parsing."""
|
||||
|
||||
def test_input_messages_parses_as_chat_request(self):
|
||||
request: EmbeddingRequest = TypeAdapter(EmbeddingRequest).validate_python(
|
||||
{
|
||||
"model": "test",
|
||||
"input": [{"role": "user", "content": "hello"}],
|
||||
"chat_template_kwargs": {"instruction": "Represent the query: "},
|
||||
}
|
||||
)
|
||||
|
||||
assert isinstance(request, EmbeddingChatInputRequest)
|
||||
assert request.input == [{"role": "user", "content": "hello"}]
|
||||
assert request.messages == [{"role": "user", "content": "hello"}]
|
||||
assert request.chat_template_kwargs == {"instruction": "Represent the query: "}
|
||||
|
||||
def test_batched_input_messages_parses_as_batch_chat_input_request(self):
|
||||
request: EmbeddingRequest = TypeAdapter(EmbeddingRequest).validate_python(
|
||||
{
|
||||
"model": "test",
|
||||
"input": [
|
||||
[{"role": "user", "content": "hello"}],
|
||||
[{"role": "user", "content": "goodbye"}],
|
||||
],
|
||||
"chat_template_kwargs": {"instruction": "Represent the query: "},
|
||||
}
|
||||
)
|
||||
|
||||
assert isinstance(request, EmbeddingBatchChatInputRequest)
|
||||
assert request.input == [
|
||||
[{"role": "user", "content": "hello"}],
|
||||
[{"role": "user", "content": "goodbye"}],
|
||||
]
|
||||
assert request.messages == [
|
||||
[{"role": "user", "content": "hello"}],
|
||||
[{"role": "user", "content": "goodbye"}],
|
||||
]
|
||||
assert request.chat_template_kwargs == {"instruction": "Represent the query: "}
|
||||
|
||||
def test_token_ids_still_parse_as_completion_request(self):
|
||||
request: EmbeddingRequest = TypeAdapter(EmbeddingRequest).validate_python(
|
||||
{
|
||||
"model": "test",
|
||||
"input": [[1, 2, 3], [4, 5]],
|
||||
}
|
||||
)
|
||||
|
||||
assert isinstance(request, EmbeddingCompletionRequest)
|
||||
assert request.input == [[1, 2, 3], [4, 5]]
|
||||
|
||||
def test_messages_still_parses_as_chat_request(self):
|
||||
request: EmbeddingRequest = TypeAdapter(EmbeddingRequest).validate_python(
|
||||
{
|
||||
"model": "test",
|
||||
"messages": [{"role": "user", "content": "hello"}],
|
||||
"chat_template_kwargs": {"instruction": "Represent the query: "},
|
||||
}
|
||||
)
|
||||
|
||||
assert isinstance(request, EmbeddingChatRequest)
|
||||
assert request.messages == [{"role": "user", "content": "hello"}]
|
||||
assert request.chat_template_kwargs == {"instruction": "Represent the query: "}
|
||||
|
||||
def test_batched_messages_parses_as_batch_chat_request(self):
|
||||
request: EmbeddingRequest = TypeAdapter(EmbeddingRequest).validate_python(
|
||||
{
|
||||
"model": "test",
|
||||
"messages": [
|
||||
[{"role": "user", "content": "hello"}],
|
||||
[{"role": "user", "content": "goodbye"}],
|
||||
],
|
||||
"chat_template_kwargs": {"instruction": "Represent the query: "},
|
||||
}
|
||||
)
|
||||
|
||||
assert isinstance(request, EmbeddingBatchChatRequest)
|
||||
assert request.messages == [
|
||||
[{"role": "user", "content": "hello"}],
|
||||
[{"role": "user", "content": "goodbye"}],
|
||||
]
|
||||
assert request.chat_template_kwargs == {"instruction": "Represent the query: "}
|
||||
|
||||
|
||||
class TestCohereEmbedRequestParsing:
|
||||
"""Unit tests for Cohere embed request parsing."""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"request_body",
|
||||
[
|
||||
{"model": "test"},
|
||||
{"model": "test", "texts": ["hello"], "images": ["image-uri"]},
|
||||
{
|
||||
"model": "test",
|
||||
"texts": ["hello"],
|
||||
"inputs": [
|
||||
{"content": [{"type": "text", "text": "hello"}]},
|
||||
],
|
||||
},
|
||||
{
|
||||
"model": "test",
|
||||
"images": ["image-uri"],
|
||||
"inputs": [
|
||||
{"content": [{"type": "text", "text": "hello"}]},
|
||||
],
|
||||
},
|
||||
{"model": "test", "texts": []},
|
||||
{"model": "test", "images": []},
|
||||
{"model": "test", "inputs": []},
|
||||
],
|
||||
)
|
||||
def test_rejects_invalid_input_field_combinations(self, request_body):
|
||||
with pytest.raises(
|
||||
ValidationError,
|
||||
match="Exactly one of texts, images, or inputs must be provided",
|
||||
):
|
||||
CohereEmbedRequest(**request_body)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"request_body",
|
||||
[
|
||||
{"model": "test", "texts": ["hello"]},
|
||||
{"model": "test", "images": ["image-uri"]},
|
||||
{
|
||||
"model": "test",
|
||||
"inputs": [
|
||||
{"content": [{"type": "text", "text": "hello"}]},
|
||||
],
|
||||
},
|
||||
{
|
||||
"model": "test",
|
||||
"inputs": [
|
||||
{
|
||||
"content": [
|
||||
{"type": "image_url", "image_url": {"url": "image-uri"}}
|
||||
]
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
)
|
||||
def test_accepts_exactly_one_non_empty_input_field(self, request_body):
|
||||
request = CohereEmbedRequest(**request_body)
|
||||
|
||||
assert request.model == "test"
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("content", "error"),
|
||||
[
|
||||
(
|
||||
{"type": "text"},
|
||||
"CohereEmbedContent with type='text' requires text",
|
||||
),
|
||||
(
|
||||
{"type": "image_url"},
|
||||
"CohereEmbedContent with type='image_url' requires image_url.url",
|
||||
),
|
||||
(
|
||||
{"type": "image_url", "image_url": {}},
|
||||
"CohereEmbedContent with type='image_url' requires image_url.url",
|
||||
),
|
||||
(
|
||||
{"type": "image_url", "image_url": {"url": ""}},
|
||||
"CohereEmbedContent with type='image_url' requires image_url.url",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_rejects_invalid_mixed_content_payloads(self, content, error):
|
||||
with pytest.raises(ValidationError, match=error):
|
||||
CohereEmbedRequest(
|
||||
model="test",
|
||||
inputs=[
|
||||
{
|
||||
"content": [content],
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
class TestResolveTruncation:
|
||||
"""Unit tests for EmbedIOProcessor._resolve_cohere_truncation."""
|
||||
|
||||
@staticmethod
|
||||
def _make_request(**kwargs) -> CohereEmbedRequest:
|
||||
defaults = {
|
||||
"model": "test",
|
||||
"input_type": "search_document",
|
||||
"texts": ["hello"],
|
||||
}
|
||||
return CohereEmbedRequest(**(defaults | kwargs))
|
||||
|
||||
def test_truncate_end_default(self):
|
||||
req = self._make_request()
|
||||
tokens, side = EmbedIOProcessor._resolve_cohere_truncation(req)
|
||||
assert tokens == -1
|
||||
assert side is None
|
||||
|
||||
def test_truncate_end_explicit(self):
|
||||
req = self._make_request(truncate="END")
|
||||
tokens, side = EmbedIOProcessor._resolve_cohere_truncation(req)
|
||||
assert tokens == -1
|
||||
assert side is None
|
||||
|
||||
def test_truncate_end_with_max_tokens(self):
|
||||
req = self._make_request(truncate="END", max_tokens=128)
|
||||
tokens, side = EmbedIOProcessor._resolve_cohere_truncation(req)
|
||||
assert tokens == 128
|
||||
assert side is None
|
||||
|
||||
def test_truncate_none(self):
|
||||
req = self._make_request(truncate="NONE")
|
||||
tokens, side = EmbedIOProcessor._resolve_cohere_truncation(req)
|
||||
assert tokens is None
|
||||
assert side is None
|
||||
|
||||
def test_truncate_none_with_max_tokens(self):
|
||||
"""truncate=NONE should NOT set truncate_prompt_tokens; the
|
||||
max_tokens limit is enforced separately via _check_max_tokens."""
|
||||
req = self._make_request(truncate="NONE", max_tokens=10)
|
||||
tokens, side = EmbedIOProcessor._resolve_cohere_truncation(req)
|
||||
assert tokens is None
|
||||
assert side is None
|
||||
|
||||
def test_truncate_start(self):
|
||||
req = self._make_request(truncate="START")
|
||||
tokens, side = EmbedIOProcessor._resolve_cohere_truncation(req)
|
||||
assert tokens == -1
|
||||
assert side == "left"
|
||||
|
||||
def test_truncate_start_with_max_tokens(self):
|
||||
req = self._make_request(truncate="START", max_tokens=64)
|
||||
tokens, side = EmbedIOProcessor._resolve_cohere_truncation(req)
|
||||
assert tokens == 64
|
||||
assert side == "left"
|
||||
|
||||
|
||||
class TestApplyStPrompt:
|
||||
"""Unit tests for EmbedIOProcessor._apply_task_instruction."""
|
||||
|
||||
@staticmethod
|
||||
def _make_handler(task_instructions: dict[str, str] | None):
|
||||
handler = object.__new__(EmbedIOProcessor)
|
||||
handler.task_instructions = task_instructions
|
||||
return handler
|
||||
|
||||
def test_no_prompts_configured(self):
|
||||
handler = self._make_handler(None)
|
||||
texts = ["hello", "world"]
|
||||
assert handler._apply_task_instruction(texts, "query") is texts
|
||||
|
||||
def test_matching_input_type(self):
|
||||
handler = self._make_handler({"query": "search_query: "})
|
||||
result = handler._apply_task_instruction(["hello"], "query")
|
||||
assert result == ["search_query: hello"]
|
||||
|
||||
def test_non_matching_input_type(self):
|
||||
handler = self._make_handler({"query": "search_query: "})
|
||||
texts = ["hello"]
|
||||
assert handler._apply_task_instruction(texts, "document") is texts
|
||||
|
||||
def test_multiple_texts(self):
|
||||
handler = self._make_handler(
|
||||
{"query": "Represent this sentence for searching: "}
|
||||
)
|
||||
result = handler._apply_task_instruction(["a", "b", "c"], "query")
|
||||
assert result == [
|
||||
"Represent this sentence for searching: a",
|
||||
"Represent this sentence for searching: b",
|
||||
"Represent this sentence for searching: c",
|
||||
]
|
||||
|
||||
def test_empty_prefix_returns_unchanged(self):
|
||||
handler = self._make_handler({"passage": ""})
|
||||
texts = ["hello"]
|
||||
assert handler._apply_task_instruction(texts, "passage") is texts
|
||||
|
||||
|
||||
class TestLoadTaskInstructions:
|
||||
"""Unit tests for EmbedIOProcessor._load_task_instructions."""
|
||||
|
||||
def test_no_attribute(self):
|
||||
class FakeConfig:
|
||||
pass
|
||||
|
||||
assert EmbedIOProcessor._load_task_instructions(FakeConfig()) is None
|
||||
|
||||
def test_with_task_instructions(self):
|
||||
class FakeConfig:
|
||||
task_instructions = {
|
||||
"retrieval.query": "Represent the query: ",
|
||||
"retrieval.passage": "",
|
||||
}
|
||||
|
||||
result = EmbedIOProcessor._load_task_instructions(FakeConfig())
|
||||
assert result == {
|
||||
"retrieval.query": "Represent the query: ",
|
||||
"retrieval.passage": "",
|
||||
}
|
||||
|
||||
def test_empty_dict(self):
|
||||
class FakeConfig:
|
||||
task_instructions = {}
|
||||
|
||||
assert EmbedIOProcessor._load_task_instructions(FakeConfig()) is None
|
||||
|
||||
def test_non_dict(self):
|
||||
class FakeConfig:
|
||||
task_instructions = "not a dict"
|
||||
|
||||
assert EmbedIOProcessor._load_task_instructions(FakeConfig()) is None
|
||||
|
||||
|
||||
class TestCheckMaxTokens:
|
||||
"""Unit tests for EmbedIOProcessor._check_cohere_max_tokens."""
|
||||
|
||||
@staticmethod
|
||||
def _fake_output(n_tokens: int):
|
||||
class _Out:
|
||||
def __init__(self, n: int):
|
||||
self.prompt_token_ids = list(range(n))
|
||||
|
||||
return _Out(n_tokens)
|
||||
|
||||
def test_none_check_is_noop(self):
|
||||
outs = [self._fake_output(100)]
|
||||
EmbedIOProcessor._check_cohere_max_tokens(outs, None)
|
||||
|
||||
def test_within_limit(self):
|
||||
outs = [self._fake_output(5), self._fake_output(3)]
|
||||
EmbedIOProcessor._check_cohere_max_tokens(outs, 5)
|
||||
|
||||
def test_exceeds_limit(self):
|
||||
outs = [self._fake_output(3), self._fake_output(10)]
|
||||
with pytest.raises(ValueError, match="exceeds max_tokens=5"):
|
||||
EmbedIOProcessor._check_cohere_max_tokens(outs, 5)
|
||||
|
||||
def test_exact_limit(self):
|
||||
outs = [self._fake_output(5)]
|
||||
EmbedIOProcessor._check_cohere_max_tokens(outs, 5)
|
||||
|
||||
|
||||
class TestValidateInputType:
|
||||
"""Unit tests for EmbedIOProcessor._validate_input_type."""
|
||||
|
||||
@staticmethod
|
||||
def _make_handler(task_instructions: dict[str, str] | None):
|
||||
handler = object.__new__(EmbedIOProcessor)
|
||||
handler.task_instructions = task_instructions
|
||||
return handler
|
||||
|
||||
def test_none_input_type_always_accepted(self):
|
||||
handler = self._make_handler(None)
|
||||
handler._validate_input_type(None)
|
||||
handler_with = self._make_handler({"query": "q: "})
|
||||
handler_with._validate_input_type(None)
|
||||
|
||||
def test_no_prompts_rejects(self):
|
||||
handler = self._make_handler(None)
|
||||
with pytest.raises(ValueError, match="does not define any input_type"):
|
||||
handler._validate_input_type("anything")
|
||||
|
||||
def test_known_type_accepted(self):
|
||||
handler = self._make_handler({"query": "q: ", "document": "d: "})
|
||||
handler._validate_input_type("query")
|
||||
handler._validate_input_type("document")
|
||||
|
||||
def test_unknown_type_rejected(self):
|
||||
handler = self._make_handler({"query": "q: ", "document": "d: "})
|
||||
with pytest.raises(ValueError, match="Unsupported input_type 'other'"):
|
||||
handler._validate_input_type("other")
|
||||
|
||||
def test_error_lists_supported(self):
|
||||
handler = self._make_handler({"a": "", "b": ""})
|
||||
with pytest.raises(ValueError, match="Supported values: a, b"):
|
||||
handler._validate_input_type("z")
|
||||
|
||||
|
||||
class TestChunkedEmbeddingProcessing:
|
||||
"""Unit tests for chunked embedding aggregation."""
|
||||
|
||||
class _FakeModelConfig:
|
||||
max_model_len = 3
|
||||
|
||||
@classmethod
|
||||
def _make_handler(cls):
|
||||
handler = object.__new__(EmbedIOProcessor)
|
||||
handler.model_config = cls._FakeModelConfig()
|
||||
handler.enable_chunked_processing = True
|
||||
return handler
|
||||
|
||||
@staticmethod
|
||||
def _make_context() -> PoolingServeContext[EmbeddingCompletionRequest]:
|
||||
request: EmbeddingRequest = TypeAdapter(EmbeddingRequest).validate_python(
|
||||
{
|
||||
"model": "test",
|
||||
"input": [[0, 1, 2, 3, 4], [10, 11]],
|
||||
}
|
||||
)
|
||||
assert isinstance(request, EmbeddingCompletionRequest)
|
||||
pooling_params = PoolingParams()
|
||||
return PoolingServeContext(
|
||||
request=request,
|
||||
pooling_params=pooling_params,
|
||||
model_name="test",
|
||||
request_id="embd-client-prompt-999-chunk-888",
|
||||
engine_inputs=[
|
||||
PoolingEngineInput(
|
||||
prompts=tokens_input(prompt_token_ids=[0, 1, 2, 3, 4]),
|
||||
params=pooling_params,
|
||||
lora_requests=None,
|
||||
priorities=0,
|
||||
),
|
||||
PoolingEngineInput(
|
||||
prompts=tokens_input(prompt_token_ids=[10, 11]),
|
||||
params=pooling_params,
|
||||
lora_requests=None,
|
||||
priorities=0,
|
||||
),
|
||||
],
|
||||
lora_request=None,
|
||||
priorities=0,
|
||||
prompt_extras=None,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _make_output(
|
||||
request_id: str,
|
||||
prompt_token_ids: list[int],
|
||||
embedding: list[float],
|
||||
) -> PoolingRequestOutput:
|
||||
return PoolingRequestOutput(
|
||||
request_id=request_id,
|
||||
outputs=PoolingOutput(data=torch.tensor(embedding)),
|
||||
prompt_token_ids=prompt_token_ids,
|
||||
num_cached_tokens=0,
|
||||
finished=True,
|
||||
)
|
||||
|
||||
def test_aggregation_uses_metadata_not_request_id_parsing(self):
|
||||
handler = self._make_handler()
|
||||
ctx = self._make_context()
|
||||
|
||||
handler.maybe_pre_process_chunked(ctx)
|
||||
|
||||
assert ctx.prompt_request_ids == [
|
||||
"embd-client-prompt-999-chunk-888-prompt-0-chunk-0",
|
||||
"embd-client-prompt-999-chunk-888-prompt-0-chunk-1",
|
||||
"embd-client-prompt-999-chunk-888-prompt-1-chunk-0",
|
||||
]
|
||||
assert ctx.chunked_embedding_metadata is not None
|
||||
assert [
|
||||
(item.prompt_index, item.chunk_index)
|
||||
for item in ctx.chunked_embedding_metadata
|
||||
] == [(0, 0), (0, 1), (1, 0)]
|
||||
|
||||
ctx.final_res_batch = [
|
||||
self._make_output(ctx.prompt_request_ids[0], [0, 1, 2], [1.0, 1.0]),
|
||||
self._make_output(ctx.prompt_request_ids[1], [3, 4], [4.0, 7.0]),
|
||||
self._make_output(ctx.prompt_request_ids[2], [10, 11], [9.0, 9.0]),
|
||||
]
|
||||
|
||||
handler._post_process_chunked(ctx)
|
||||
|
||||
assert len(ctx.final_res_batch) == 2
|
||||
assert ctx.final_res_batch[0].request_id == (
|
||||
"embd-client-prompt-999-chunk-888-prompt-0"
|
||||
)
|
||||
assert ctx.final_res_batch[0].prompt_token_ids == [0, 1, 2, 3, 4]
|
||||
assert torch.allclose(
|
||||
ctx.final_res_batch[0].outputs.data,
|
||||
torch.tensor([2.2, 3.4]),
|
||||
)
|
||||
assert ctx.final_res_batch[1].request_id == (
|
||||
"embd-client-prompt-999-chunk-888-prompt-1"
|
||||
)
|
||||
assert ctx.final_res_batch[1].prompt_token_ids == [10, 11]
|
||||
assert torch.allclose(
|
||||
ctx.final_res_batch[1].outputs.data,
|
||||
torch.tensor([9.0, 9.0]),
|
||||
)
|
||||
@@ -1,101 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import weakref
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm import LLM, EmbeddingRequestOutput, PoolingParams
|
||||
from vllm.inputs import PromptType
|
||||
from vllm.tasks import PoolingTask
|
||||
|
||||
MODEL_NAME = "intfloat/multilingual-e5-small"
|
||||
|
||||
prompt = "The chef prepared a delicious meal."
|
||||
prompt_token_ids = [0, 581, 21861, 133888, 10, 8, 150, 60744, 109911, 5, 2]
|
||||
embedding_size = 384
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def llm(vllm_runner):
|
||||
with vllm_runner(
|
||||
MODEL_NAME,
|
||||
max_model_len=None,
|
||||
max_num_batched_tokens=32768,
|
||||
tensor_parallel_size=1,
|
||||
gpu_memory_utilization=0.75,
|
||||
enforce_eager=True,
|
||||
seed=0,
|
||||
enable_chunked_prefill=None,
|
||||
) as runner:
|
||||
assert embedding_size == runner.llm.model_config.embedding_size
|
||||
# pytest caches yielded fixtures until after teardown, so use a proxy to
|
||||
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
|
||||
yield weakref.proxy(runner.llm)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_str_prompts(llm: LLM):
|
||||
outputs = llm.embed(prompt, use_tqdm=False)
|
||||
assert len(outputs) == 1
|
||||
assert isinstance(outputs[0], EmbeddingRequestOutput)
|
||||
assert outputs[0].prompt_token_ids == prompt_token_ids
|
||||
assert len(outputs[0].outputs.embedding) == embedding_size
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_token_ids_prompts(llm: LLM):
|
||||
outputs = llm.embed([prompt_token_ids], use_tqdm=False)
|
||||
assert len(outputs) == 1
|
||||
assert isinstance(outputs[0], EmbeddingRequestOutput)
|
||||
assert outputs[0].prompt_token_ids == prompt_token_ids
|
||||
assert len(outputs[0].outputs.embedding) == embedding_size
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_list_prompts(llm: LLM):
|
||||
prompts: list[PromptType] = [prompt, prompt_token_ids]
|
||||
outputs = llm.embed(prompts, use_tqdm=False)
|
||||
assert len(outputs) == 2
|
||||
for i in range(len(outputs)):
|
||||
assert isinstance(outputs[i], EmbeddingRequestOutput)
|
||||
assert outputs[i].prompt_token_ids == prompt_token_ids
|
||||
assert len(outputs[i].outputs.embedding) == embedding_size
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_pooling_params(llm: LLM):
|
||||
def get_outputs(normalize):
|
||||
outputs = llm.embed(
|
||||
[prompt],
|
||||
pooling_params=PoolingParams(use_activation=normalize),
|
||||
use_tqdm=False,
|
||||
)
|
||||
return torch.tensor([x.outputs.embedding for x in outputs])
|
||||
|
||||
default = get_outputs(normalize=None)
|
||||
w_normal = get_outputs(normalize=True)
|
||||
wo_normal = get_outputs(normalize=False)
|
||||
|
||||
assert torch.allclose(default, w_normal, atol=1e-2), "Default should use normal."
|
||||
assert not torch.allclose(w_normal, wo_normal, atol=1e-2), (
|
||||
"wo_normal should not use normal."
|
||||
)
|
||||
assert torch.allclose(w_normal, F.normalize(wo_normal, p=2, dim=-1), atol=1e-2), (
|
||||
"w_normal should be close to normal(wo_normal)."
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"task", ["token_classify", "classify", "token_embed", "plugin"]
|
||||
)
|
||||
def test_unsupported_tasks(llm: LLM, task: PoolingTask):
|
||||
if task == "plugin":
|
||||
err_msg = "No IOProcessor plugin installed."
|
||||
elif task == "token_embed":
|
||||
err_msg = "Try switching the model's pooling_task via.+"
|
||||
else:
|
||||
err_msg = "Classification API is not supported by this model.+"
|
||||
with pytest.raises(ValueError, match=err_msg):
|
||||
llm.encode(prompt, pooling_task=task, use_tqdm=False)
|
||||
@@ -1,801 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
|
||||
import numpy as np
|
||||
import openai
|
||||
import pybase64 as base64
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
import requests
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from tests.models.language.pooling.embed_utils import (
|
||||
float_embeddings,
|
||||
run_embedding_correctness_test,
|
||||
)
|
||||
from tests.models.utils import check_embeddings_close
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.entrypoints.pooling.embed.protocol import EmbeddingResponse
|
||||
from vllm.entrypoints.pooling.pooling.protocol import PoolingResponse
|
||||
from vllm.entrypoints.pooling.utils import (
|
||||
MetadataItem,
|
||||
build_metadata_items,
|
||||
decode_pooling_output,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.renderers.chat_utils import ChatCompletionMessageParam
|
||||
from vllm.tokenizers import get_tokenizer
|
||||
from vllm.utils.serial_utils import EMBED_DTYPES, ENDIANNESS, binary2tensor
|
||||
|
||||
MODEL_NAME = "intfloat/multilingual-e5-small"
|
||||
DUMMY_CHAT_TEMPLATE = """{% for message in messages %}{{message['role'] + ': ' + message['content'] + '\\n'}}{% endfor %}""" # noqa: E501
|
||||
DTYPE = "bfloat16"
|
||||
input_text = "The best thing about vLLM is that it supports many different models"
|
||||
input_tokens = [
|
||||
0,
|
||||
581,
|
||||
2965,
|
||||
13580,
|
||||
1672,
|
||||
81,
|
||||
23708,
|
||||
594,
|
||||
83,
|
||||
450,
|
||||
442,
|
||||
8060,
|
||||
7,
|
||||
5941,
|
||||
12921,
|
||||
115774,
|
||||
2,
|
||||
]
|
||||
|
||||
# On ROCm, floating-point reductions in attention and GEMM kernels are
|
||||
# non-associative and sensitive to batch geometry. Force LLM instances
|
||||
# into an identical, deterministic execution mode:
|
||||
ROCM_DETERMINISM_ARGS: list[str] = (
|
||||
["--max-num-seqs", "1"] if current_platform.is_rocm() else []
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
args = [
|
||||
"--runner",
|
||||
"pooling",
|
||||
"--dtype",
|
||||
DTYPE,
|
||||
"--enforce-eager",
|
||||
"--max-model-len",
|
||||
"512",
|
||||
"--chat-template",
|
||||
DUMMY_CHAT_TEMPLATE,
|
||||
*ROCM_DETERMINISM_ARGS,
|
||||
]
|
||||
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def client(server):
|
||||
async with server.get_async_client() as async_client:
|
||||
yield async_client
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def hf_model(hf_runner):
|
||||
with hf_runner(MODEL_NAME, dtype=DTYPE, is_sentence_transformer=True) as hf_model:
|
||||
yield hf_model
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
async def test_negative_token_ids(client: openai.AsyncOpenAI, model_name: str):
|
||||
# A negative token id is out of vocabulary just like an over-large one, but
|
||||
# is not caught by the upper-bound check. Unlike the OpenAI completion
|
||||
# schema, the pooling schema does not constrain token ids to be
|
||||
# non-negative, so the request reaches the shared engine-level validation.
|
||||
with pytest.raises(openai.BadRequestError, match=".*out of vocabulary.*"):
|
||||
await client.embeddings.create(model=model_name, input=[-1])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
async def test_basic(
|
||||
server: RemoteOpenAIServer, client: openai.AsyncOpenAI, model_name: str
|
||||
):
|
||||
# test /v1/models
|
||||
response = requests.get(server.url_for("/v1/models"))
|
||||
model = response.json()["data"][0]["id"]
|
||||
assert model == MODEL_NAME
|
||||
|
||||
models = await client.models.list()
|
||||
models = models.data
|
||||
served_model = models[0]
|
||||
assert served_model.id == MODEL_NAME
|
||||
|
||||
# test /tokenize
|
||||
response = requests.post(
|
||||
server.url_for("/tokenize"),
|
||||
json={"model": model_name, "prompt": input_text},
|
||||
)
|
||||
assert response.json()["tokens"] == input_tokens
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
async def test_completion_request(
|
||||
client: openai.AsyncOpenAI, model_name: str, hf_model
|
||||
):
|
||||
# test input: str
|
||||
embedding_response = await client.embeddings.create(
|
||||
model=model_name,
|
||||
input=input_text,
|
||||
encoding_format="float",
|
||||
)
|
||||
embeddings = EmbeddingResponse.model_validate(
|
||||
embedding_response.model_dump(mode="json")
|
||||
)
|
||||
|
||||
assert embeddings.id is not None
|
||||
assert len(embeddings.data) == 1
|
||||
assert len(embeddings.data[0].embedding) == 384
|
||||
assert embeddings.usage.completion_tokens == 0
|
||||
assert embeddings.usage.prompt_tokens == len(input_tokens)
|
||||
assert embeddings.usage.total_tokens == len(input_tokens)
|
||||
|
||||
vllm_outputs = float_embeddings(embeddings)
|
||||
run_embedding_correctness_test(hf_model, [input_text], vllm_outputs)
|
||||
|
||||
# test input: list[int]
|
||||
embedding_response = await client.embeddings.create(
|
||||
model=model_name,
|
||||
input=input_tokens,
|
||||
encoding_format="float",
|
||||
)
|
||||
embeddings = EmbeddingResponse.model_validate(
|
||||
embedding_response.model_dump(mode="json")
|
||||
)
|
||||
|
||||
assert embeddings.id is not None
|
||||
assert len(embeddings.data) == 1
|
||||
assert len(embeddings.data[0].embedding) == 384
|
||||
assert embeddings.usage.completion_tokens == 0
|
||||
assert embeddings.usage.prompt_tokens == len(input_tokens)
|
||||
assert embeddings.usage.total_tokens == len(input_tokens)
|
||||
|
||||
vllm_outputs = float_embeddings(embeddings)
|
||||
run_embedding_correctness_test(hf_model, [input_text], vllm_outputs)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
async def test_completion_request_batched(
|
||||
client: openai.AsyncOpenAI, model_name: str, hf_model
|
||||
):
|
||||
N = 10
|
||||
input_texts = [input_text] * N
|
||||
|
||||
# test input: list[str]
|
||||
embedding_response = await client.embeddings.create(
|
||||
model=model_name,
|
||||
input=input_texts,
|
||||
encoding_format="float",
|
||||
)
|
||||
embeddings = EmbeddingResponse.model_validate(
|
||||
embedding_response.model_dump(mode="json")
|
||||
)
|
||||
|
||||
assert embeddings.id is not None
|
||||
assert len(embeddings.data) == N
|
||||
assert len(embeddings.data[0].embedding) == 384
|
||||
assert embeddings.usage.completion_tokens == 0
|
||||
assert embeddings.usage.prompt_tokens == len(input_tokens) * N
|
||||
assert embeddings.usage.total_tokens == len(input_tokens) * N
|
||||
|
||||
vllm_outputs = float_embeddings(embeddings)
|
||||
run_embedding_correctness_test(hf_model, input_texts, vllm_outputs)
|
||||
|
||||
# test list[list[int]]
|
||||
embedding_response = await client.embeddings.create(
|
||||
model=model_name,
|
||||
input=[input_tokens] * N,
|
||||
encoding_format="float",
|
||||
)
|
||||
embeddings = EmbeddingResponse.model_validate(
|
||||
embedding_response.model_dump(mode="json")
|
||||
)
|
||||
|
||||
assert embeddings.id is not None
|
||||
assert len(embeddings.data) == N
|
||||
assert len(embeddings.data[0].embedding) == 384
|
||||
assert embeddings.usage.completion_tokens == 0
|
||||
assert embeddings.usage.prompt_tokens == len(input_tokens) * N
|
||||
assert embeddings.usage.total_tokens == len(input_tokens) * N
|
||||
|
||||
vllm_outputs = float_embeddings(embeddings)
|
||||
run_embedding_correctness_test(hf_model, input_texts, vllm_outputs)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
async def test_truncate_prompt_tokens(client: openai.AsyncOpenAI, model_name: str):
|
||||
input_texts = [
|
||||
"Como o Brasil pode fomentar o desenvolvimento de modelos de IA?",
|
||||
]
|
||||
|
||||
# test single embedding
|
||||
embedding_response = await client.embeddings.create(
|
||||
model=model_name, input=input_texts, extra_body={"truncate_prompt_tokens": 10}
|
||||
)
|
||||
embeddings = EmbeddingResponse.model_validate(
|
||||
embedding_response.model_dump(mode="json")
|
||||
)
|
||||
|
||||
assert embeddings.id is not None
|
||||
assert len(embeddings.data) == 1
|
||||
assert len(embeddings.data[0].embedding) == 384
|
||||
assert embeddings.usage.completion_tokens == 0
|
||||
assert embeddings.usage.prompt_tokens == 10
|
||||
assert embeddings.usage.total_tokens == 10
|
||||
|
||||
input_tokens = [
|
||||
1,
|
||||
24428,
|
||||
289,
|
||||
18341,
|
||||
26165,
|
||||
285,
|
||||
19323,
|
||||
283,
|
||||
289,
|
||||
26789,
|
||||
3871,
|
||||
28728,
|
||||
9901,
|
||||
340,
|
||||
2229,
|
||||
385,
|
||||
340,
|
||||
315,
|
||||
28741,
|
||||
28804,
|
||||
2,
|
||||
]
|
||||
embedding_response = await client.embeddings.create(
|
||||
model=model_name, input=input_tokens, extra_body={"truncate_prompt_tokens": 10}
|
||||
)
|
||||
embeddings = EmbeddingResponse.model_validate(
|
||||
embedding_response.model_dump(mode="json")
|
||||
)
|
||||
|
||||
assert embeddings.id is not None
|
||||
assert len(embeddings.data) == 1
|
||||
assert len(embeddings.data[0].embedding) == 384
|
||||
assert embeddings.usage.completion_tokens == 0
|
||||
assert embeddings.usage.prompt_tokens == 10
|
||||
assert embeddings.usage.total_tokens == 10
|
||||
|
||||
# invalid_truncate_prompt_tokens
|
||||
input_texts = [
|
||||
"Como o Brasil pode fomentar o desenvolvimento de modelos de IA?",
|
||||
]
|
||||
|
||||
with pytest.raises(openai.BadRequestError):
|
||||
response = await client.embeddings.create(
|
||||
model=model_name,
|
||||
input=input_texts,
|
||||
extra_body={"truncate_prompt_tokens": 8193},
|
||||
)
|
||||
assert "error" in response.object
|
||||
assert (
|
||||
"truncate_prompt_tokens value is greater than max_model_len. "
|
||||
"Please request a smaller truncation size." in response.message
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
async def test_padding(client: openai.AsyncOpenAI, model_name: str):
|
||||
input_texts = ["Hello"]
|
||||
|
||||
async def prompt_tokens(**extra_body):
|
||||
response = await client.embeddings.create(
|
||||
model=model_name,
|
||||
input=input_texts,
|
||||
extra_body=extra_body,
|
||||
)
|
||||
embeddings = EmbeddingResponse.model_validate(response.model_dump(mode="json"))
|
||||
assert len(embeddings.data) == 1
|
||||
return embeddings.usage.prompt_tokens
|
||||
|
||||
unpadded = await prompt_tokens()
|
||||
padded = await prompt_tokens(padding="max_length")
|
||||
|
||||
# Models trained with a fixed sequence length and no attention mask need
|
||||
# this; without it their embeddings are not comparable.
|
||||
assert padded > unpadded
|
||||
assert await prompt_tokens(padding="do_not_pad") == unpadded
|
||||
|
||||
with pytest.raises(openai.BadRequestError):
|
||||
await client.embeddings.create(
|
||||
model=model_name,
|
||||
input=input_texts,
|
||||
extra_body={"padding": "longest"},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
async def test_chat_request(
|
||||
server: RemoteOpenAIServer, client: openai.AsyncOpenAI, model_name: str
|
||||
):
|
||||
messages: list[ChatCompletionMessageParam] = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "The cat sat on the mat.",
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "A feline was resting on a rug.",
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Stars twinkle brightly in the night sky.",
|
||||
},
|
||||
]
|
||||
|
||||
# test chat request basic usage
|
||||
chat_response = requests.post(
|
||||
server.url_for("v1/embeddings"),
|
||||
json={
|
||||
"model": model_name,
|
||||
"messages": messages,
|
||||
"encoding_format": "float",
|
||||
},
|
||||
)
|
||||
chat_response.raise_for_status()
|
||||
chat_embeddings = EmbeddingResponse.model_validate(chat_response.json())
|
||||
|
||||
tokenizer = get_tokenizer(tokenizer_name=model_name)
|
||||
prompt = tokenizer.apply_chat_template(
|
||||
messages,
|
||||
chat_template=DUMMY_CHAT_TEMPLATE,
|
||||
add_generation_prompt=True,
|
||||
continue_final_message=False,
|
||||
tokenize=False,
|
||||
)
|
||||
completion_response = await client.embeddings.create(
|
||||
model=model_name,
|
||||
input=prompt,
|
||||
encoding_format="float",
|
||||
# To be consistent with chat
|
||||
extra_body={"add_special_tokens": False},
|
||||
)
|
||||
completion_embeddings = EmbeddingResponse.model_validate(
|
||||
completion_response.model_dump(mode="json")
|
||||
)
|
||||
|
||||
assert chat_embeddings.id is not None
|
||||
assert completion_embeddings.id is not None
|
||||
assert chat_embeddings.created <= completion_embeddings.created
|
||||
# Use tolerance-based comparison for embeddings
|
||||
check_embeddings_close(
|
||||
embeddings_0_lst=float_embeddings(chat_embeddings),
|
||||
embeddings_1_lst=float_embeddings(completion_embeddings),
|
||||
name_0="chat",
|
||||
name_1="completion",
|
||||
)
|
||||
assert chat_embeddings.model_dump(exclude={"id", "created", "data"}) == (
|
||||
completion_embeddings.model_dump(exclude={"id", "created", "data"})
|
||||
)
|
||||
|
||||
# test add_generation_prompt
|
||||
response = requests.post(
|
||||
server.url_for("v1/embeddings"),
|
||||
json={"model": model_name, "messages": messages, "add_generation_prompt": True},
|
||||
)
|
||||
|
||||
response.raise_for_status()
|
||||
output = EmbeddingResponse.model_validate(response.json())
|
||||
|
||||
assert output.object == "list"
|
||||
assert len(output.data) == 1
|
||||
assert output.model == MODEL_NAME
|
||||
assert output.usage.prompt_tokens == 33
|
||||
|
||||
# test continue_final_message
|
||||
response = requests.post(
|
||||
server.url_for("v1/embeddings"),
|
||||
json={
|
||||
"model": model_name,
|
||||
"messages": messages,
|
||||
"continue_final_message": True,
|
||||
},
|
||||
)
|
||||
|
||||
response.raise_for_status()
|
||||
output = EmbeddingResponse.model_validate(response.json())
|
||||
|
||||
assert output.object == "list"
|
||||
assert len(output.data) == 1
|
||||
assert output.model == MODEL_NAME
|
||||
assert output.usage.prompt_tokens == 33
|
||||
|
||||
# test add_special_tokens
|
||||
response = requests.post(
|
||||
server.url_for("v1/embeddings"),
|
||||
json={"model": model_name, "messages": messages, "add_special_tokens": True},
|
||||
)
|
||||
|
||||
response.raise_for_status()
|
||||
output = EmbeddingResponse.model_validate(response.json())
|
||||
|
||||
assert output.object == "list"
|
||||
assert len(output.data) == 1
|
||||
assert output.model == MODEL_NAME
|
||||
assert output.usage.prompt_tokens == 35
|
||||
|
||||
# test continue_final_message with add_generation_prompt
|
||||
response = requests.post(
|
||||
server.url_for("v1/embeddings"),
|
||||
json={
|
||||
"model": model_name,
|
||||
"messages": messages,
|
||||
"continue_final_message": True,
|
||||
"add_generation_prompt": True,
|
||||
},
|
||||
)
|
||||
assert (
|
||||
"Cannot set both `continue_final_message` and `add_generation_prompt` to True."
|
||||
in response.json()["error"]["message"]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invocations_completion_request(
|
||||
server: RemoteOpenAIServer, client: openai.AsyncOpenAI
|
||||
):
|
||||
request_args = {
|
||||
"model": MODEL_NAME,
|
||||
"input": input_text,
|
||||
"encoding_format": "float",
|
||||
}
|
||||
|
||||
completion_response = await client.embeddings.create(**request_args)
|
||||
|
||||
invocation_response = requests.post(
|
||||
server.url_for("invocations"), json=request_args
|
||||
)
|
||||
invocation_response.raise_for_status()
|
||||
|
||||
completion_output = completion_response.model_dump()
|
||||
invocation_output = invocation_response.json()
|
||||
|
||||
assert completion_output.keys() == invocation_output.keys()
|
||||
for completion_data, invocation_data in zip(
|
||||
completion_output["data"], invocation_output["data"]
|
||||
):
|
||||
assert completion_data.keys() == invocation_data.keys()
|
||||
check_embeddings_close(
|
||||
embeddings_0_lst=[completion_data["embedding"]],
|
||||
embeddings_1_lst=[invocation_data["embedding"]],
|
||||
name_0="completion",
|
||||
name_1="invocation",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invocations_chat_request(server: RemoteOpenAIServer):
|
||||
messages: list[ChatCompletionMessageParam] = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "The cat sat on the mat.",
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "A feline was resting on a rug.",
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Stars twinkle brightly in the night sky.",
|
||||
},
|
||||
]
|
||||
|
||||
request_args = {
|
||||
"model": MODEL_NAME,
|
||||
"messages": messages,
|
||||
"encoding_format": "float",
|
||||
}
|
||||
|
||||
chat_response = requests.post(server.url_for("v1/embeddings"), json=request_args)
|
||||
chat_response.raise_for_status()
|
||||
|
||||
invocation_response = requests.post(
|
||||
server.url_for("invocations"), json=request_args
|
||||
)
|
||||
invocation_response.raise_for_status()
|
||||
|
||||
chat_output = chat_response.json()
|
||||
invocation_output = invocation_response.json()
|
||||
|
||||
assert chat_output.keys() == invocation_output.keys()
|
||||
for chat_data, invocation_data in zip(
|
||||
chat_output["data"], invocation_output["data"]
|
||||
):
|
||||
assert chat_data.keys() == invocation_data.keys()
|
||||
check_embeddings_close(
|
||||
embeddings_0_lst=[chat_data["embedding"]],
|
||||
embeddings_1_lst=[invocation_data["embedding"]],
|
||||
name_0="chat",
|
||||
name_1="invocation",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
async def test_base64_embedding(hf_model, client: openai.AsyncOpenAI, model_name: str):
|
||||
input_texts = [
|
||||
"Hello my name is",
|
||||
"The best thing about vLLM is that it supports many different models",
|
||||
]
|
||||
|
||||
def check_outputs(outputs: list[list[float]]) -> None:
|
||||
# The embeddings endpoint executes list elements as separate scheduler
|
||||
# requests, so use the same per-request geometry for the HF reference.
|
||||
for input_text, output in zip(input_texts, outputs, strict=True):
|
||||
run_embedding_correctness_test(hf_model, [input_text], [output])
|
||||
|
||||
responses_float = await client.embeddings.create(
|
||||
input=input_texts, model=model_name, encoding_format="float"
|
||||
)
|
||||
float_data = [d.embedding for d in responses_float.data]
|
||||
check_outputs(float_data)
|
||||
|
||||
responses_base64 = await client.embeddings.create(
|
||||
input=input_texts, model=model_name, encoding_format="base64"
|
||||
)
|
||||
base64_data = []
|
||||
for data in responses_base64.data:
|
||||
base64_data.append(
|
||||
np.frombuffer(base64.b64decode(data.embedding), dtype="float32").tolist()
|
||||
)
|
||||
|
||||
check_outputs(base64_data)
|
||||
|
||||
# Default response is float32 decoded from base64 by OpenAI Client
|
||||
responses_default = await client.embeddings.create(
|
||||
input=input_texts, model=model_name
|
||||
)
|
||||
default_data = [d.embedding for d in responses_default.data]
|
||||
check_outputs(default_data)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
async def test_base64_embed_dtype_and_endianness(
|
||||
server: RemoteOpenAIServer, client: openai.AsyncOpenAI, model_name: str
|
||||
):
|
||||
input_texts = [input_text] * 3
|
||||
responses_float = await client.embeddings.create(
|
||||
input=input_texts, model=model_name, encoding_format="float"
|
||||
)
|
||||
float_data = [d.embedding for d in responses_float.data]
|
||||
|
||||
for embed_dtype in EMBED_DTYPES:
|
||||
for endianness in ENDIANNESS:
|
||||
responses_base64 = requests.post(
|
||||
server.url_for("/v1/embeddings"),
|
||||
json={
|
||||
"model": model_name,
|
||||
"input": input_texts,
|
||||
"encoding_format": "base64",
|
||||
"embed_dtype": embed_dtype,
|
||||
"endianness": endianness,
|
||||
},
|
||||
)
|
||||
|
||||
base64_data = []
|
||||
for data in responses_base64.json()["data"]:
|
||||
binary = base64.b64decode(data["embedding"])
|
||||
tensor = binary2tensor(binary, (-1,), embed_dtype, endianness)
|
||||
base64_data.append(tensor.to(torch.float32).tolist())
|
||||
|
||||
check_embeddings_close(
|
||||
embeddings_0_lst=float_data,
|
||||
embeddings_1_lst=base64_data,
|
||||
name_0="float_data",
|
||||
name_1="base64_data",
|
||||
tol=1e-2,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
async def test_bytes_embed_dtype_and_endianness(
|
||||
server: RemoteOpenAIServer, client: openai.AsyncOpenAI, model_name: str
|
||||
):
|
||||
input_texts = [input_text] * 3
|
||||
responses_float = await client.embeddings.create(
|
||||
input=input_texts, model=model_name, encoding_format="float"
|
||||
)
|
||||
float_data = [d.embedding for d in responses_float.data]
|
||||
|
||||
for embed_dtype in EMBED_DTYPES:
|
||||
for endianness in ENDIANNESS:
|
||||
responses_bytes = requests.post(
|
||||
server.url_for("/v1/embeddings"),
|
||||
json={
|
||||
"model": model_name,
|
||||
"input": input_texts,
|
||||
"encoding_format": "bytes",
|
||||
"embed_dtype": embed_dtype,
|
||||
"endianness": endianness,
|
||||
},
|
||||
)
|
||||
|
||||
metadata = json.loads(responses_bytes.headers["metadata"])
|
||||
body = responses_bytes.content
|
||||
items = [MetadataItem(**x) for x in metadata["data"]]
|
||||
|
||||
bytes_data = decode_pooling_output(items=items, body=body)
|
||||
bytes_data = [x.to(torch.float32).tolist() for x in bytes_data]
|
||||
|
||||
check_embeddings_close(
|
||||
embeddings_0_lst=float_data,
|
||||
embeddings_1_lst=bytes_data,
|
||||
name_0="float_data",
|
||||
name_1="bytes_data",
|
||||
tol=1e-2,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
async def test_bytes_only_embed_dtype_and_endianness(
|
||||
server: RemoteOpenAIServer, client: openai.AsyncOpenAI, model_name: str
|
||||
):
|
||||
input_texts = [
|
||||
"The best thing about vLLM is that it supports many different models",
|
||||
] * 2
|
||||
|
||||
responses_float = await client.embeddings.create(
|
||||
input=input_texts, model=model_name, encoding_format="float"
|
||||
)
|
||||
float_data = [d.embedding for d in responses_float.data]
|
||||
embedding_size = len(float_data[0])
|
||||
|
||||
for embed_dtype in EMBED_DTYPES:
|
||||
for endianness in ENDIANNESS:
|
||||
responses_bytes = requests.post(
|
||||
server.url_for("/v1/embeddings"),
|
||||
json={
|
||||
"model": model_name,
|
||||
"input": input_texts,
|
||||
"encoding_format": "bytes_only",
|
||||
"embed_dtype": embed_dtype,
|
||||
"endianness": endianness,
|
||||
},
|
||||
)
|
||||
|
||||
assert "metadata" not in responses_bytes.headers
|
||||
body = responses_bytes.content
|
||||
items = build_metadata_items(
|
||||
embed_dtype=embed_dtype,
|
||||
endianness=endianness,
|
||||
shape=(embedding_size,),
|
||||
n_request=len(input_texts),
|
||||
)
|
||||
|
||||
bytes_data = decode_pooling_output(items=items, body=body)
|
||||
bytes_data = [x.to(torch.float32).tolist() for x in bytes_data]
|
||||
|
||||
check_embeddings_close(
|
||||
embeddings_0_lst=float_data,
|
||||
embeddings_1_lst=bytes_data,
|
||||
name_0="float_data",
|
||||
name_1="bytes_data",
|
||||
tol=1e-2,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
@pytest.mark.parametrize("param_name", ["encoding_format", "embed_dtype", "endianness"])
|
||||
async def test_params_not_supported(
|
||||
server: RemoteOpenAIServer, model_name: str, param_name: str
|
||||
):
|
||||
responses_base64 = requests.post(
|
||||
server.url_for("/v1/embeddings"),
|
||||
json={
|
||||
"model": model_name,
|
||||
"input": input_text,
|
||||
"encoding_format": "base64",
|
||||
param_name: f"bad_{param_name}",
|
||||
},
|
||||
)
|
||||
|
||||
assert responses_base64.status_code == 400
|
||||
assert "literal_error" in responses_base64.json()["error"]["message"]
|
||||
assert f"bad_{param_name}" in responses_base64.json()["error"]["message"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
async def test_use_activation(server: RemoteOpenAIServer, model_name: str):
|
||||
async def get_outputs(use_activation):
|
||||
request_args = {
|
||||
"model": MODEL_NAME,
|
||||
"input": input_text,
|
||||
"encoding_format": "float",
|
||||
"use_activation": use_activation,
|
||||
}
|
||||
|
||||
response = requests.post(server.url_for("v1/embeddings"), json=request_args)
|
||||
outputs = response.json()
|
||||
|
||||
return torch.tensor([x["embedding"] for x in outputs["data"]])
|
||||
|
||||
default = await get_outputs(use_activation=None)
|
||||
w_normal = await get_outputs(use_activation=True)
|
||||
wo_normal = await get_outputs(use_activation=False)
|
||||
|
||||
assert torch.allclose(default, w_normal, atol=1e-2), "Default should use normal."
|
||||
assert not torch.allclose(w_normal, wo_normal, atol=1e-2), (
|
||||
"wo_normal should not use normal."
|
||||
)
|
||||
assert torch.allclose(w_normal, F.normalize(wo_normal, p=2, dim=-1), atol=1e-2), (
|
||||
"w_normal should be close to normal(wo_normal)."
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
async def test_pooling_embed(server: RemoteOpenAIServer, model_name: str):
|
||||
task = "embed"
|
||||
response = requests.post(
|
||||
server.url_for("pooling"),
|
||||
json={
|
||||
"model": model_name,
|
||||
"input": input_text,
|
||||
"encoding_format": "float",
|
||||
"task": task,
|
||||
},
|
||||
)
|
||||
|
||||
poolings = PoolingResponse.model_validate(response.json())
|
||||
|
||||
assert len(poolings.data) == 1
|
||||
assert len(poolings.data[0].data) == 384
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
@pytest.mark.parametrize(
|
||||
"task", ["classify", "token_classify", "token_embed", "plugin"]
|
||||
)
|
||||
async def test_pooling_not_supported(
|
||||
server: RemoteOpenAIServer, model_name: str, task: str
|
||||
):
|
||||
response = requests.post(
|
||||
server.url_for("pooling"),
|
||||
json={
|
||||
"model": model_name,
|
||||
"input": "test",
|
||||
"encoding_format": "float",
|
||||
"task": task,
|
||||
},
|
||||
)
|
||||
assert response.json()["error"]["type"] == "BadRequestError"
|
||||
if task == "plugin":
|
||||
err_msg = "No IOProcessor plugin installed."
|
||||
elif task == "token_embed":
|
||||
err_msg = "Try switching the model's pooling_task via"
|
||||
else:
|
||||
err_msg = f"Unsupported task: {task!r}"
|
||||
assert response.json()["error"]["message"].startswith(err_msg)
|
||||
@@ -1,127 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Run `pytest tests/entrypoints/openai/test_embedding_dimensions.py`."""
|
||||
|
||||
import openai
|
||||
import pytest
|
||||
|
||||
from tests.conftest import HfRunner
|
||||
from tests.models.language.pooling.embed_utils import (
|
||||
float_embeddings,
|
||||
run_embedding_correctness_test,
|
||||
)
|
||||
from tests.models.utils import EmbedModelInfo
|
||||
from tests.utils import ROCM_EXTRA_ARGS, RemoteOpenAIServer
|
||||
from vllm.entrypoints.pooling.embed.protocol import EmbeddingResponse
|
||||
|
||||
MODELS = [
|
||||
EmbedModelInfo("intfloat/multilingual-e5-small", is_matryoshka=False),
|
||||
EmbedModelInfo(
|
||||
"Snowflake/snowflake-arctic-embed-m-v1.5",
|
||||
is_matryoshka=True,
|
||||
matryoshka_dimensions=[256],
|
||||
),
|
||||
]
|
||||
|
||||
input_texts = [
|
||||
"The chef prepared a delicious meal.",
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", params=MODELS)
|
||||
def model_info(request):
|
||||
return request.param
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", params=["bfloat16"])
|
||||
def dtype(request):
|
||||
return request.param
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server(model_info, dtype: str):
|
||||
args = [
|
||||
"--runner",
|
||||
"pooling",
|
||||
# use half precision for speed and memory savings in CI environment
|
||||
"--dtype",
|
||||
dtype,
|
||||
"--enforce-eager",
|
||||
"--max-model-len",
|
||||
"512",
|
||||
] + ROCM_EXTRA_ARGS
|
||||
|
||||
if model_info.name == "Snowflake/snowflake-arctic-embed-m-v1.5":
|
||||
# Manually enable Matryoshka Embeddings
|
||||
args.extend(
|
||||
["--trust_remote_code", "--hf_overrides", '{"matryoshka_dimensions":[256]}']
|
||||
)
|
||||
|
||||
with RemoteOpenAIServer(model_info.name, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def hf_model(hf_runner, model_info, dtype: str):
|
||||
with hf_runner(
|
||||
model_info.name, dtype=dtype, is_sentence_transformer=True
|
||||
) as hf_model:
|
||||
yield hf_model
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_matryoshka(
|
||||
model_info: EmbedModelInfo, server: RemoteOpenAIServer, hf_model: HfRunner
|
||||
):
|
||||
client = server.get_async_client()
|
||||
|
||||
async def make_request_and_correctness_test(dimensions):
|
||||
prompts = input_texts * 3
|
||||
|
||||
embedding_response = await client.embeddings.create(
|
||||
model=model_info.name,
|
||||
input=prompts,
|
||||
dimensions=dimensions,
|
||||
encoding_format="float",
|
||||
)
|
||||
embeddings = EmbeddingResponse.model_validate(
|
||||
embedding_response.model_dump(mode="json")
|
||||
)
|
||||
|
||||
assert embeddings.id is not None
|
||||
assert len(embeddings.data) == 3
|
||||
assert len(embeddings.data[0].embedding) > 0
|
||||
assert embeddings.usage.completion_tokens == 0
|
||||
assert embeddings.usage.prompt_tokens > 0
|
||||
assert embeddings.usage.total_tokens > 0
|
||||
|
||||
if dimensions is not None:
|
||||
assert len(embeddings.data[0].embedding) == dimensions
|
||||
|
||||
vllm_outputs = float_embeddings(embeddings)
|
||||
run_embedding_correctness_test(hf_model, prompts, vllm_outputs, dimensions)
|
||||
|
||||
if model_info.is_matryoshka:
|
||||
valid_dimensions: list[int | None] = [None]
|
||||
if model_info.matryoshka_dimensions is not None:
|
||||
valid_dimensions += model_info.matryoshka_dimensions[:2]
|
||||
|
||||
for dimensions in valid_dimensions:
|
||||
await make_request_and_correctness_test(dimensions)
|
||||
|
||||
invalid_dimensions: list[int | None] = [-1]
|
||||
if model_info.matryoshka_dimensions is not None:
|
||||
assert 5 not in model_info.matryoshka_dimensions
|
||||
invalid_dimensions.append(5)
|
||||
|
||||
for dimensions in invalid_dimensions:
|
||||
with pytest.raises(openai.BadRequestError):
|
||||
await make_request_and_correctness_test(dimensions)
|
||||
|
||||
else:
|
||||
for dimensions in [None]:
|
||||
await make_request_and_correctness_test(dimensions)
|
||||
|
||||
for dimensions in [-1, 16]:
|
||||
with pytest.raises(openai.BadRequestError):
|
||||
await make_request_and_correctness_test(dimensions)
|
||||
@@ -1,452 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Test cases for long text embedding with automatic chunking mechanism.
|
||||
|
||||
This test suite validates vLLM's automatic chunking functionality for handling
|
||||
text inputs that exceed the model's maximum token length, specifically targeting
|
||||
the intfloat/multilingual-e5-small model (max token length: 512).
|
||||
"""
|
||||
|
||||
import random
|
||||
|
||||
import openai
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.entrypoints.pooling.embed.protocol import EmbeddingResponse
|
||||
|
||||
|
||||
def _generate_random_text(word_count: int) -> str:
|
||||
"""Generate deterministic text with approximately the specified word count."""
|
||||
rng = random.Random(word_count)
|
||||
|
||||
# Common English words with focus on verbs and nouns for realistic text
|
||||
common_words = [
|
||||
# Essential articles and pronouns (minimal)
|
||||
"the",
|
||||
"and",
|
||||
"you",
|
||||
"they",
|
||||
"this",
|
||||
"that",
|
||||
"these",
|
||||
"those",
|
||||
# Action verbs
|
||||
"create",
|
||||
"build",
|
||||
"develop",
|
||||
"design",
|
||||
"implement",
|
||||
"execute",
|
||||
"analyze",
|
||||
"process",
|
||||
"generate",
|
||||
"calculate",
|
||||
"evaluate",
|
||||
"optimize",
|
||||
"transform",
|
||||
"integrate",
|
||||
"configure",
|
||||
"deploy",
|
||||
"monitor",
|
||||
"manage",
|
||||
"discover",
|
||||
"explore",
|
||||
"investigate",
|
||||
"research",
|
||||
"study",
|
||||
"examine",
|
||||
"improve",
|
||||
"enhance",
|
||||
"upgrade",
|
||||
"modify",
|
||||
"update",
|
||||
"maintain",
|
||||
"solve",
|
||||
"resolve",
|
||||
"handle",
|
||||
"address",
|
||||
"tackle",
|
||||
"overcome",
|
||||
"communicate",
|
||||
"collaborate",
|
||||
"coordinate",
|
||||
"organize",
|
||||
"plan",
|
||||
"achieve",
|
||||
"accomplish",
|
||||
"complete",
|
||||
"finish",
|
||||
"deliver",
|
||||
"provide",
|
||||
# Technology and science nouns
|
||||
"system",
|
||||
"application",
|
||||
"software",
|
||||
"hardware",
|
||||
"network",
|
||||
"database",
|
||||
"algorithm",
|
||||
"model",
|
||||
"framework",
|
||||
"platform",
|
||||
"interface",
|
||||
"protocol",
|
||||
"architecture",
|
||||
"infrastructure",
|
||||
"component",
|
||||
"module",
|
||||
"service",
|
||||
"technology",
|
||||
"innovation",
|
||||
"solution",
|
||||
"methodology",
|
||||
"approach",
|
||||
"artificial",
|
||||
"intelligence",
|
||||
"machine",
|
||||
"learning",
|
||||
"neural",
|
||||
"network",
|
||||
"computer",
|
||||
"processor",
|
||||
"memory",
|
||||
"storage",
|
||||
"computation",
|
||||
"data",
|
||||
"information",
|
||||
"knowledge",
|
||||
"insight",
|
||||
"pattern",
|
||||
"trend",
|
||||
"analysis",
|
||||
"research",
|
||||
"development",
|
||||
"engineering",
|
||||
"science",
|
||||
"mathematics",
|
||||
"statistics",
|
||||
"probability",
|
||||
"optimization",
|
||||
"performance",
|
||||
"efficiency",
|
||||
# General nouns
|
||||
"project",
|
||||
"team",
|
||||
"organization",
|
||||
"company",
|
||||
"business",
|
||||
"industry",
|
||||
"market",
|
||||
"customer",
|
||||
"user",
|
||||
"client",
|
||||
"product",
|
||||
"feature",
|
||||
"function",
|
||||
"requirement",
|
||||
"specification",
|
||||
"documentation",
|
||||
"report",
|
||||
"result",
|
||||
"outcome",
|
||||
"impact",
|
||||
"benefit",
|
||||
"advantage",
|
||||
"challenge",
|
||||
"problem",
|
||||
"opportunity",
|
||||
"strategy",
|
||||
"goal",
|
||||
"objective",
|
||||
"target",
|
||||
"milestone",
|
||||
"process",
|
||||
"procedure",
|
||||
"workflow",
|
||||
"pipeline",
|
||||
"operation",
|
||||
"task",
|
||||
"activity",
|
||||
"event",
|
||||
"session",
|
||||
"meeting",
|
||||
"discussion",
|
||||
"decision",
|
||||
]
|
||||
|
||||
words = []
|
||||
for _ in range(word_count):
|
||||
words.append(rng.choice(common_words))
|
||||
|
||||
# Add some punctuation for more realistic text
|
||||
text = " ".join(words)
|
||||
# Add periods every 10-20 words
|
||||
words_list = text.split()
|
||||
result = []
|
||||
for i, word in enumerate(words_list):
|
||||
result.append(word)
|
||||
if (i + 1) % rng.randint(10, 20) == 0 and i < len(words_list) - 1:
|
||||
result[-1] += "."
|
||||
|
||||
return " ".join(result)
|
||||
|
||||
|
||||
MODEL_NAME = "intfloat/multilingual-e5-small"
|
||||
DTYPE = "bfloat16"
|
||||
|
||||
# Test text: Generate text with approximately 1500 words to exceed 1024 tokens
|
||||
LONG_TEXT_1500_WORDS = _generate_random_text(1500)
|
||||
|
||||
# Test text: Generate text with approximately 2500 words to exceed 2048 tokens
|
||||
LONG_TEXT_2500_WORDS = _generate_random_text(2500)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server_with_chunked_processing():
|
||||
"""Start server with automatic chunking processing enabled."""
|
||||
args = [
|
||||
"--runner",
|
||||
"pooling",
|
||||
"--dtype",
|
||||
DTYPE,
|
||||
"--enforce-eager",
|
||||
"--max-model-len",
|
||||
"512", # Set smaller max_model_len to trigger chunking mechanism
|
||||
"--pooler-config",
|
||||
(
|
||||
'{"pooling_type": "MEAN", "use_activation": true, '
|
||||
'"enable_chunked_processing": true, "max_embed_len": 10000}'
|
||||
),
|
||||
"--gpu-memory-utilization",
|
||||
"0.8",
|
||||
]
|
||||
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def client_with_chunked_processing(server_with_chunked_processing):
|
||||
"""Create async client with chunking processing support."""
|
||||
async with server_with_chunked_processing.get_async_client() as async_client:
|
||||
yield async_client
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
async def test_long_text_embedding_1500_chars(
|
||||
client_with_chunked_processing: openai.AsyncOpenAI, model_name: str
|
||||
):
|
||||
"""Test embedding processing for ~1500 character long text
|
||||
(~1028 tokens, exceeding 512 token limit)."""
|
||||
# Verify text length
|
||||
# Verify text has sufficient word count (approximately 1500 words)
|
||||
word_count = len(LONG_TEXT_1500_WORDS.split())
|
||||
assert word_count >= 1400, f"Test text word count insufficient: {word_count} words"
|
||||
|
||||
# Send embedding request
|
||||
embedding_response = await client_with_chunked_processing.embeddings.create(
|
||||
model=model_name,
|
||||
input=[LONG_TEXT_1500_WORDS],
|
||||
encoding_format="float",
|
||||
)
|
||||
|
||||
# Verify response structure
|
||||
embeddings = EmbeddingResponse.model_validate(
|
||||
embedding_response.model_dump(mode="json")
|
||||
)
|
||||
|
||||
assert embeddings.id is not None
|
||||
assert len(embeddings.data) == 1
|
||||
assert (
|
||||
len(embeddings.data[0].embedding) == 384
|
||||
) # multilingual-e5-small embedding dimension
|
||||
assert embeddings.usage.completion_tokens == 0
|
||||
# Due to chunked processing, token count should
|
||||
# reflect actual processed tokens
|
||||
# With ~1500 words, we expect roughly
|
||||
# 1024+ tokens (exceeding 512 token limit)
|
||||
# Should exceed single chunk limit of 512
|
||||
assert embeddings.usage.prompt_tokens > 800
|
||||
assert embeddings.usage.total_tokens == embeddings.usage.prompt_tokens
|
||||
|
||||
# Verify embedding vector validity
|
||||
embedding_vector = embeddings.data[0].embedding
|
||||
assert isinstance(embedding_vector, list)
|
||||
assert all(isinstance(x, float) for x in embedding_vector), (
|
||||
"Embedding vector should contain floats"
|
||||
)
|
||||
assert not all(x == 0 for x in embedding_vector), (
|
||||
"Embedding vector should not be all zeros"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
async def test_long_text_embedding_2500_chars(
|
||||
client_with_chunked_processing: openai.AsyncOpenAI, model_name: str
|
||||
):
|
||||
"""Test embedding processing for ~2500 character long text
|
||||
(~2048 tokens, requiring multiple chunks)."""
|
||||
# Verify text length
|
||||
# Verify text has sufficient word count (approximately 2500 words)
|
||||
word_count = len(LONG_TEXT_2500_WORDS.split())
|
||||
assert word_count >= 2300, f"Test text word count insufficient: {word_count} words"
|
||||
|
||||
# Send embedding request
|
||||
embedding_response = await client_with_chunked_processing.embeddings.create(
|
||||
model=model_name,
|
||||
input=[LONG_TEXT_2500_WORDS],
|
||||
encoding_format="float",
|
||||
)
|
||||
|
||||
# Verify response structure
|
||||
embeddings = EmbeddingResponse.model_validate(
|
||||
embedding_response.model_dump(mode="json")
|
||||
)
|
||||
|
||||
assert embeddings.id is not None
|
||||
assert len(embeddings.data) == 1
|
||||
assert (
|
||||
len(embeddings.data[0].embedding) == 384
|
||||
) # multilingual-e5-small embedding dimension
|
||||
assert embeddings.usage.completion_tokens == 0
|
||||
# Due to chunked processing, token count should
|
||||
# reflect actual processed tokens
|
||||
# With ~2500 words, we expect
|
||||
# roughly 2048+ tokens (requiring multiple chunks)
|
||||
# Should require multiple chunks for processing
|
||||
assert embeddings.usage.prompt_tokens > 1500
|
||||
assert embeddings.usage.total_tokens == embeddings.usage.prompt_tokens
|
||||
|
||||
# Verify embedding vector validity
|
||||
embedding_vector = embeddings.data[0].embedding
|
||||
assert isinstance(embedding_vector, list)
|
||||
assert all(isinstance(x, float) for x in embedding_vector), (
|
||||
"Embedding vector should contain floats"
|
||||
)
|
||||
assert not all(x == 0 for x in embedding_vector), (
|
||||
"Embedding vector should not be all zeros"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
async def test_batch_long_text_embedding(
|
||||
client_with_chunked_processing: openai.AsyncOpenAI, model_name: str
|
||||
):
|
||||
"""Test batch long text embedding processing."""
|
||||
input_texts = [
|
||||
LONG_TEXT_1500_WORDS,
|
||||
LONG_TEXT_2500_WORDS,
|
||||
"This is a short text test.", # Short text for comparison
|
||||
]
|
||||
|
||||
# Send batch embedding request
|
||||
embedding_response = await client_with_chunked_processing.embeddings.create(
|
||||
model=model_name,
|
||||
input=input_texts,
|
||||
encoding_format="float",
|
||||
)
|
||||
|
||||
# Verify response structure
|
||||
embeddings = EmbeddingResponse.model_validate(
|
||||
embedding_response.model_dump(mode="json")
|
||||
)
|
||||
|
||||
assert embeddings.id is not None
|
||||
assert len(embeddings.data) == 3 # Three input texts
|
||||
|
||||
# Verify each embedding dimension
|
||||
for i, embedding_data in enumerate(embeddings.data):
|
||||
assert len(embedding_data.embedding) == 384
|
||||
assert embedding_data.index == i
|
||||
|
||||
# Verify embedding vector validity
|
||||
embedding_vector = embedding_data.embedding
|
||||
assert all(isinstance(x, float) for x in embedding_vector)
|
||||
assert not all(x == 0 for x in embedding_vector)
|
||||
|
||||
# Verify token usage
|
||||
assert embeddings.usage.completion_tokens == 0
|
||||
# Total token count should be very substantial
|
||||
assert embeddings.usage.prompt_tokens > 1000
|
||||
assert embeddings.usage.total_tokens == embeddings.usage.prompt_tokens
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
async def test_chunked_vs_normal_consistency(
|
||||
client_with_chunked_processing: openai.AsyncOpenAI, model_name: str
|
||||
):
|
||||
"""Test consistency between chunked and
|
||||
normal processing (using short text)."""
|
||||
# Use a short text within the 512 token limit
|
||||
short_text = (
|
||||
"Artificial intelligence technology is changing our world, "
|
||||
"bringing unprecedented opportunities and challenges."
|
||||
)
|
||||
|
||||
# Send embedding request
|
||||
embedding_response = await client_with_chunked_processing.embeddings.create(
|
||||
model=model_name,
|
||||
input=[short_text],
|
||||
encoding_format="float",
|
||||
)
|
||||
|
||||
# Verify response structure
|
||||
embeddings = EmbeddingResponse.model_validate(
|
||||
embedding_response.model_dump(mode="json")
|
||||
)
|
||||
|
||||
assert embeddings.id is not None
|
||||
assert len(embeddings.data) == 1
|
||||
assert len(embeddings.data[0].embedding) == 384
|
||||
assert embeddings.usage.completion_tokens == 0
|
||||
# Short text should not require chunked processing
|
||||
assert embeddings.usage.prompt_tokens < 512
|
||||
assert embeddings.usage.total_tokens == embeddings.usage.prompt_tokens
|
||||
|
||||
# 验证embedding向量的有效性
|
||||
embedding_vector = embeddings.data[0].embedding
|
||||
assert isinstance(embedding_vector, list)
|
||||
assert all(isinstance(x, float) for x in embedding_vector)
|
||||
assert not all(x == 0 for x in embedding_vector)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
async def test_chunked_processing_response_format(
|
||||
client_with_chunked_processing: openai.AsyncOpenAI, model_name: str
|
||||
):
|
||||
"""Test response format and structure during chunked processing."""
|
||||
# Test with long text to trigger chunking
|
||||
embedding_response = await client_with_chunked_processing.embeddings.create(
|
||||
model=model_name,
|
||||
input=[LONG_TEXT_1500_WORDS],
|
||||
encoding_format="float",
|
||||
)
|
||||
|
||||
# Verify response structure
|
||||
embeddings = EmbeddingResponse.model_validate(
|
||||
embedding_response.model_dump(mode="json")
|
||||
)
|
||||
|
||||
assert embeddings.id is not None
|
||||
assert len(embeddings.data) == 1
|
||||
assert embeddings.data[0].object == "embedding"
|
||||
assert embeddings.data[0].index == 0
|
||||
|
||||
# Verify embedding vector properties
|
||||
embedding_vector = embeddings.data[0].embedding
|
||||
assert isinstance(embedding_vector, list)
|
||||
import math
|
||||
|
||||
vector_norm = math.sqrt(sum(x * x for x in embedding_vector))
|
||||
# Check that the vector is normalized
|
||||
# (default behavior for most embedding models)
|
||||
assert 0.8 < vector_norm < 1.2, (
|
||||
f"Vector norm should be reasonable, actual: {vector_norm}"
|
||||
)
|
||||
@@ -1,211 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
from transformers import AutoProcessor
|
||||
|
||||
from tests.utils import VLLM_PATH, RemoteOpenAIServer
|
||||
from vllm.entrypoints.pooling.embed.protocol import EmbeddingResponse
|
||||
from vllm.multimodal.media import MediaWithBytes
|
||||
from vllm.multimodal.utils import encode_image_url, fetch_image
|
||||
|
||||
MODEL_NAME = "TIGER-Lab/VLM2Vec-Full"
|
||||
MAXIMUM_IMAGES = 2
|
||||
|
||||
vlm2vec_jinja_path = VLLM_PATH / "examples/pooling/embed/template/vlm2vec_phi3v.jinja"
|
||||
assert vlm2vec_jinja_path.exists()
|
||||
|
||||
# Test different image extensions (JPG/PNG) and formats (gray/RGB/RGBA)
|
||||
TEST_IMAGE_ASSETS = [
|
||||
"2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
|
||||
"Grayscale_8bits_palette_sample_image.png", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/Grayscale_8bits_palette_sample_image.png",
|
||||
"1280px-Venn_diagram_rgb.svg.png", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/1280px-Venn_diagram_rgb.svg.png",
|
||||
"RGBA_comp.png", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/RGBA_comp.png",
|
||||
]
|
||||
|
||||
input_text = "The best thing about vLLM is that it supports many different models"
|
||||
image_url = "https://vllm-public-assets.s3.us-west-2.amazonaws.com/multimodal_asset/cat_snow.jpg"
|
||||
image_base64 = {"url": encode_image_url(fetch_image(image_url))}
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
args = [
|
||||
"--runner",
|
||||
"pooling",
|
||||
"--max-model-len",
|
||||
"2048",
|
||||
"--max-num-seqs",
|
||||
"5",
|
||||
"--enforce-eager",
|
||||
"--trust-remote-code",
|
||||
"--limit-mm-per-prompt",
|
||||
json.dumps({"image": MAXIMUM_IMAGES}),
|
||||
"--chat-template",
|
||||
str(vlm2vec_jinja_path),
|
||||
"--trust-request-mm-kwargs",
|
||||
]
|
||||
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
def test_chat_text_request(server: RemoteOpenAIServer, model_name: str):
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": input_text,
|
||||
},
|
||||
]
|
||||
|
||||
# note: vlm2vec_phi3v.jinja
|
||||
# Embedding models should only embed one message at a time.
|
||||
|
||||
response = requests.post(
|
||||
server.url_for("v1/embeddings"),
|
||||
json={"model": model_name, "messages": messages},
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
output = EmbeddingResponse.model_validate(response.json())
|
||||
assert len(output.data) == 1
|
||||
assert output.model == MODEL_NAME
|
||||
assert len(output.data[0].embedding) == 3072
|
||||
assert output.usage.prompt_tokens == 14
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
def test_chat_image_url_request(server: RemoteOpenAIServer, model_name: str):
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Represent the user's input."},
|
||||
{"type": "image_url", "image_url": {"url": image_url}},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
response = requests.post(
|
||||
server.url_for("v1/embeddings"),
|
||||
json={"model": model_name, "messages": messages},
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
output = EmbeddingResponse.model_validate(response.json())
|
||||
assert len(output.data) == 1
|
||||
assert output.model == MODEL_NAME
|
||||
assert len(output.data[0].embedding) == 3072
|
||||
assert output.usage.prompt_tokens == 767
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
def test_chat_image_base64_request(server: RemoteOpenAIServer, model_name: str):
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Represent the user's input."},
|
||||
{"type": "image_url", "image_url": image_base64},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
response = requests.post(
|
||||
server.url_for("v1/embeddings"),
|
||||
json={"model": model_name, "messages": messages},
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
output = EmbeddingResponse.model_validate(response.json())
|
||||
assert len(output.data) == 1
|
||||
assert output.model == MODEL_NAME
|
||||
assert len(output.data[0].embedding) == 3072
|
||||
assert output.usage.prompt_tokens == 767
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
def test_chat_image_with_media_io_kwargs(server: RemoteOpenAIServer, model_name: str):
|
||||
rgba_image_url = (
|
||||
"https://vllm-public-assets.s3.us-west-2.amazonaws.com"
|
||||
"/vision_model_images/RGBA_comp.png"
|
||||
)
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Represent the user's input."},
|
||||
{"type": "image_url", "image_url": {"url": rgba_image_url}},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
response = requests.post(
|
||||
server.url_for("v1/embeddings"),
|
||||
json={
|
||||
"model": model_name,
|
||||
"messages": messages,
|
||||
"media_io_kwargs": {
|
||||
"image": {"rgba_background_color": [0, 0, 0]},
|
||||
},
|
||||
},
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
output = EmbeddingResponse.model_validate(response.json())
|
||||
assert len(output.data) == 1
|
||||
assert len(output.data[0].embedding) == 3072
|
||||
|
||||
|
||||
def get_hf_prompt_tokens(model_name, content, image_url):
|
||||
processor = AutoProcessor.from_pretrained(
|
||||
model_name, trust_remote_code=True, num_crops=4
|
||||
)
|
||||
|
||||
placeholder = "<|image_1|> "
|
||||
prompt = f"{placeholder}{content}"
|
||||
image = fetch_image(image_url)
|
||||
# Unwrap MediaWithBytes if present
|
||||
if isinstance(image, MediaWithBytes):
|
||||
image = image.media
|
||||
images = [image]
|
||||
inputs = processor(prompt, images, return_tensors="pt")
|
||||
return inputs.input_ids.shape[1]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
@pytest.mark.parametrize("image_url", TEST_IMAGE_ASSETS, indirect=True)
|
||||
async def test_image_embedding(
|
||||
server: RemoteOpenAIServer, model_name: str, image_url: str
|
||||
):
|
||||
content_text = "Represent the given image."
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "image_url", "image_url": {"url": image_url}},
|
||||
{"type": "text", "text": content_text},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
response = requests.post(
|
||||
server.url_for("v1/embeddings"),
|
||||
json={"model": model_name, "messages": messages, "encoding_format": "float"},
|
||||
)
|
||||
response.raise_for_status()
|
||||
embeddings = EmbeddingResponse.model_validate(response.json())
|
||||
|
||||
hf_prompt_tokens = get_hf_prompt_tokens(model_name, content_text, image_url)
|
||||
|
||||
assert embeddings.id is not None
|
||||
assert len(embeddings.data) == 1
|
||||
assert len(embeddings.data[0].embedding) == 3072
|
||||
assert embeddings.usage.completion_tokens == 0
|
||||
assert embeddings.usage.prompt_tokens == hf_prompt_tokens
|
||||
assert embeddings.usage.total_tokens == hf_prompt_tokens
|
||||
@@ -1,223 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for Cohere embed protocol: build_typed_embeddings and its
|
||||
underlying packing helpers, plus Cohere-specific serving helpers."""
|
||||
|
||||
import struct
|
||||
from unittest.mock import Mock
|
||||
|
||||
import numpy as np
|
||||
import pybase64 as base64
|
||||
import pytest
|
||||
|
||||
from vllm.config import ModelConfig, PoolerConfig
|
||||
from vllm.entrypoints.pooling.embed.protocol import (
|
||||
EmbeddingCompletionRequest,
|
||||
build_typed_embeddings,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_embeddings() -> list[list[float]]:
|
||||
return [
|
||||
[0.1, -0.2, 0.3, -0.4, 0.5, -0.6, 0.7, -0.8],
|
||||
[-0.05, 0.15, -0.25, 0.35, -0.45, 0.55, -0.65, 0.75],
|
||||
]
|
||||
|
||||
|
||||
class TestBuildTypedEmbeddingsFloat:
|
||||
def test_float_passthrough(self, sample_embeddings: list[list[float]]):
|
||||
result = build_typed_embeddings(sample_embeddings, ["float"])
|
||||
assert result.float == sample_embeddings
|
||||
assert result.binary is None
|
||||
|
||||
def test_empty_input(self):
|
||||
result = build_typed_embeddings([], ["float"])
|
||||
assert result.float == []
|
||||
|
||||
|
||||
class TestBuildTypedEmbeddingsBinary:
|
||||
def test_binary_packing(self):
|
||||
# 8 values: positive->1, negative->0 => bits: 10101010 = 0xAA = 170
|
||||
# signed: 170 - 128 = 42
|
||||
embs = [[1.0, -1.0, 1.0, -1.0, 1.0, -1.0, 1.0, -1.0]]
|
||||
result = build_typed_embeddings(embs, ["binary"])
|
||||
assert result.binary is not None
|
||||
assert result.binary[0] == [42]
|
||||
|
||||
def test_ubinary_packing(self):
|
||||
embs = [[1.0, -1.0, 1.0, -1.0, 1.0, -1.0, 1.0, -1.0]]
|
||||
result = build_typed_embeddings(embs, ["ubinary"])
|
||||
assert result.ubinary is not None
|
||||
assert result.ubinary[0] == [170] # 0b10101010
|
||||
|
||||
def test_binary_all_positive(self):
|
||||
embs = [[0.1] * 8]
|
||||
result = build_typed_embeddings(embs, ["binary"])
|
||||
assert result.binary is not None
|
||||
# all bits = 1 => 0xFF = 255, signed: 255 - 128 = 127
|
||||
assert result.binary[0] == [127]
|
||||
|
||||
def test_binary_all_negative(self):
|
||||
embs = [[-0.1] * 8]
|
||||
result = build_typed_embeddings(embs, ["binary"])
|
||||
assert result.binary is not None
|
||||
# all bits = 0, signed: 0 - 128 = -128
|
||||
assert result.binary[0] == [-128]
|
||||
|
||||
def test_binary_dimension_is_eighth(self, sample_embeddings: list[list[float]]):
|
||||
result = build_typed_embeddings(sample_embeddings, ["binary"])
|
||||
assert result.binary is not None
|
||||
for orig, packed in zip(sample_embeddings, result.binary):
|
||||
assert len(packed) == len(orig) // 8
|
||||
|
||||
def test_zero_treated_as_positive(self):
|
||||
embs = [[0.0] * 8]
|
||||
result = build_typed_embeddings(embs, ["binary"])
|
||||
assert result.binary is not None
|
||||
# 0.0 >= 0 is True, so bit=1 for all => 127 (signed)
|
||||
assert result.binary[0] == [127]
|
||||
|
||||
def test_negative_zero_treated_as_positive(self):
|
||||
"""Packing follows ``value >= 0``, not the IEEE-754 sign bit.
|
||||
|
||||
``-0.0 >= 0`` is True, so these bits must be set. An implementation
|
||||
reading the sign bit instead would pack -128 here.
|
||||
"""
|
||||
result = build_typed_embeddings([[-0.0] * 8], ["binary"])
|
||||
assert result.binary is not None
|
||||
assert result.binary[0] == [127]
|
||||
|
||||
def test_negative_denormal_treated_as_negative(self):
|
||||
"""Sign is decided at float64 precision.
|
||||
|
||||
The smallest float64 denormal underflows to -0.0 in float32, where
|
||||
``>= 0`` is True; narrowing the comparison would wrongly pack 127.
|
||||
"""
|
||||
result = build_typed_embeddings([[-5e-324] * 8], ["binary"])
|
||||
assert result.binary is not None
|
||||
assert result.binary[0] == [-128]
|
||||
|
||||
def test_empty_input(self):
|
||||
result = build_typed_embeddings([], ["binary", "ubinary"])
|
||||
assert result.binary == []
|
||||
assert result.ubinary == []
|
||||
|
||||
@pytest.mark.parametrize("embs", [[0.1] * 8, [[[0.1] * 8] * 2]], ids=["1d", "3d"])
|
||||
def test_non_2d_input_raises(self, embs):
|
||||
"""A malformed batch must raise rather than pack along the last axis.
|
||||
|
||||
Packing a 1D or 3D input would silently emit wrongly shaped
|
||||
``binary``/``ubinary`` values instead of reporting an error.
|
||||
"""
|
||||
with pytest.raises(ValueError, match="2D batch"):
|
||||
build_typed_embeddings(embs, ["binary"])
|
||||
|
||||
def test_non_multiple_of_8_raises(self):
|
||||
embs = [[0.1] * 7]
|
||||
with pytest.raises(ValueError, match="multiple of 8"):
|
||||
build_typed_embeddings(embs, ["binary"])
|
||||
|
||||
def test_ubinary_non_multiple_of_8_raises(self):
|
||||
embs = [[0.1] * 10]
|
||||
with pytest.raises(ValueError, match="multiple of 8"):
|
||||
build_typed_embeddings(embs, ["ubinary"])
|
||||
|
||||
|
||||
class TestBuildTypedEmbeddingsBase64:
|
||||
def test_base64_roundtrip(self, sample_embeddings: list[list[float]]):
|
||||
result = build_typed_embeddings(sample_embeddings, ["base64"])
|
||||
assert result.base64 is not None
|
||||
assert len(result.base64) == 2
|
||||
|
||||
for orig, b64_str in zip(sample_embeddings, result.base64):
|
||||
decoded = base64.b64decode(b64_str)
|
||||
n = len(orig)
|
||||
values = struct.unpack(f"<{n}f", decoded)
|
||||
np.testing.assert_allclose(orig, values, rtol=1e-5)
|
||||
|
||||
def test_base64_byte_length(self):
|
||||
embs = [[0.1, 0.2, 0.3]]
|
||||
result = build_typed_embeddings(embs, ["base64"])
|
||||
assert result.base64 is not None
|
||||
raw = base64.b64decode(result.base64[0])
|
||||
assert len(raw) == 3 * 4 # 3 floats * 4 bytes each
|
||||
|
||||
|
||||
class TestBuildTypedEmbeddingsMultiple:
|
||||
def test_all_types_at_once(self, sample_embeddings: list[list[float]]):
|
||||
result = build_typed_embeddings(
|
||||
sample_embeddings,
|
||||
["float", "binary", "ubinary", "base64"],
|
||||
)
|
||||
assert result.float is not None
|
||||
assert result.binary is not None
|
||||
assert result.ubinary is not None
|
||||
assert result.base64 is not None
|
||||
|
||||
def test_subset_types(self, sample_embeddings: list[list[float]]):
|
||||
result = build_typed_embeddings(sample_embeddings, ["float", "binary"])
|
||||
assert result.float is not None
|
||||
assert result.binary is not None
|
||||
assert result.ubinary is None
|
||||
assert result.base64 is None
|
||||
|
||||
def test_unknown_type_ignored(self, sample_embeddings: list[list[float]]):
|
||||
result = build_typed_embeddings(sample_embeddings, ["float", "unknown_type"])
|
||||
assert result.float is not None
|
||||
|
||||
|
||||
def _embedding_model_config(
|
||||
*,
|
||||
max_model_len: int = 128,
|
||||
max_embed_len: int | None = None,
|
||||
enable_chunked_processing: bool = False,
|
||||
) -> Mock:
|
||||
model_config = Mock(spec=ModelConfig)
|
||||
model_config.max_model_len = max_model_len
|
||||
model_config.encoder_config = None
|
||||
model_config.pooler_config = PoolerConfig(
|
||||
seq_pooling_type="CLS",
|
||||
max_embed_len=max_embed_len,
|
||||
enable_chunked_processing=enable_chunked_processing,
|
||||
)
|
||||
return model_config
|
||||
|
||||
|
||||
class TestMaxEmbedLenCapsInput:
|
||||
"""``build_tok_params`` used to enforce ``max_embed_len`` indirectly, by
|
||||
setting ``max_output_tokens = max_model_len - max_embed_len`` and relying on
|
||||
``max_input_tokens == max_total_tokens - max_output_tokens``. The cap is now
|
||||
expressed directly as ``max_total_tokens``."""
|
||||
|
||||
def test_non_chunked_max_embed_len_caps_input(self):
|
||||
cfg = _embedding_model_config(max_model_len=128, max_embed_len=64)
|
||||
|
||||
tok_params = EmbeddingCompletionRequest(model="m", input="hi").build_tok_params(
|
||||
cfg
|
||||
)
|
||||
|
||||
assert tok_params.max_input_tokens == 64
|
||||
assert tok_params.max_output_tokens == 0
|
||||
|
||||
def test_chunked_max_embed_len_caps_input(self):
|
||||
cfg = _embedding_model_config(
|
||||
max_model_len=128, max_embed_len=64, enable_chunked_processing=True
|
||||
)
|
||||
|
||||
tok_params = EmbeddingCompletionRequest(model="m", input="hi").build_tok_params(
|
||||
cfg
|
||||
)
|
||||
|
||||
assert tok_params.max_input_tokens == 64
|
||||
assert tok_params.max_output_tokens == 0
|
||||
|
||||
def test_no_max_embed_len_falls_back_to_model_len(self):
|
||||
cfg = _embedding_model_config(max_model_len=128)
|
||||
|
||||
tok_params = EmbeddingCompletionRequest(model="m", input="hi").build_tok_params(
|
||||
cfg
|
||||
)
|
||||
|
||||
assert tok_params.max_input_tokens == 128
|
||||
assert tok_params.max_output_tokens == 0
|
||||
@@ -1,64 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import weakref
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.models.utils import softmax
|
||||
from vllm import LLM, PoolingParams
|
||||
|
||||
MODEL_NAME = "internlm/internlm2-1_8b-reward"
|
||||
|
||||
prompts = ["The chef prepared a delicious meal."]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def llm(vllm_runner):
|
||||
with vllm_runner(
|
||||
MODEL_NAME,
|
||||
max_model_len=None,
|
||||
max_num_batched_tokens=32768,
|
||||
tensor_parallel_size=1,
|
||||
gpu_memory_utilization=0.75,
|
||||
enforce_eager=True,
|
||||
trust_remote_code=True,
|
||||
seed=0,
|
||||
enable_chunked_prefill=None,
|
||||
) as runner:
|
||||
# pytest caches yielded fixtures until after teardown, so use a proxy to
|
||||
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
|
||||
yield weakref.proxy(runner.llm)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_config(llm: LLM):
|
||||
vllm_config = llm.llm_engine.vllm_config
|
||||
assert vllm_config.cache_config.enable_prefix_caching
|
||||
assert vllm_config.scheduler_config.enable_chunked_prefill
|
||||
|
||||
|
||||
def test_pooling_params(llm: LLM):
|
||||
def get_outputs(use_activation):
|
||||
outputs = llm.encode(
|
||||
prompts,
|
||||
pooling_params=PoolingParams(use_activation=use_activation),
|
||||
pooling_task="token_classify",
|
||||
use_tqdm=False,
|
||||
)
|
||||
return torch.cat([x.outputs.data for x in outputs])
|
||||
|
||||
default = get_outputs(use_activation=None)
|
||||
w_activation = get_outputs(use_activation=True)
|
||||
wo_activation = get_outputs(use_activation=False)
|
||||
|
||||
assert torch.allclose(default, w_activation, atol=1e-2), (
|
||||
"Default should use activation."
|
||||
)
|
||||
assert not torch.allclose(w_activation, wo_activation, atol=1e-2), (
|
||||
"wo_activation should not use activation."
|
||||
)
|
||||
assert torch.allclose(softmax(wo_activation), w_activation, atol=1e-2), (
|
||||
"w_activation should be close to activation(wo_activation)."
|
||||
)
|
||||
@@ -1,570 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
|
||||
import numpy as np
|
||||
import pybase64 as base64
|
||||
import pytest
|
||||
import requests
|
||||
import torch
|
||||
|
||||
from tests.models.utils import check_embeddings_close
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.entrypoints.pooling.pooling.protocol import PoolingResponse
|
||||
from vllm.entrypoints.pooling.utils import (
|
||||
MetadataItem,
|
||||
build_metadata_items,
|
||||
decode_pooling_output,
|
||||
)
|
||||
from vllm.renderers.chat_utils import ChatCompletionMessageParam
|
||||
from vllm.tokenizers import get_tokenizer
|
||||
from vllm.utils.serial_utils import EMBED_DTYPES, ENDIANNESS, binary2tensor
|
||||
|
||||
MODEL_NAME = "internlm/internlm2-1_8b-reward"
|
||||
DUMMY_CHAT_TEMPLATE = """{% for message in messages %}{{message['role'] + ': ' + message['content'] + '\\n'}}{% endfor %}""" # noqa: E501
|
||||
input_text = "The chef prepared a delicious meal."
|
||||
input_tokens = [1, 918, 29981, 10166, 395, 18067, 15265, 281]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
args = [
|
||||
"--runner",
|
||||
"pooling",
|
||||
# use half precision for speed and memory savings in CI environment
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--enforce-eager",
|
||||
"--max-model-len",
|
||||
"512",
|
||||
"--chat-template",
|
||||
DUMMY_CHAT_TEMPLATE,
|
||||
"--trust-remote-code",
|
||||
]
|
||||
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
def test_basic(server: RemoteOpenAIServer, model_name: str):
|
||||
# test /v1/models
|
||||
response = requests.get(server.url_for("/v1/models"))
|
||||
served_model = response.json()["data"][0]["id"]
|
||||
assert served_model == MODEL_NAME
|
||||
|
||||
# test /tokenize
|
||||
response = requests.post(
|
||||
server.url_for("/tokenize"),
|
||||
json={"model": model_name, "prompt": input_text},
|
||||
)
|
||||
assert response.json()["tokens"] == input_tokens
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
def test_completion_request(server: RemoteOpenAIServer, model_name: str):
|
||||
# test input: str
|
||||
response = requests.post(
|
||||
server.url_for("pooling"),
|
||||
json={"model": model_name, "input": input_text, "encoding_format": "float"},
|
||||
)
|
||||
response.raise_for_status()
|
||||
poolings = PoolingResponse.model_validate(response.json())
|
||||
|
||||
assert poolings.id is not None
|
||||
assert len(poolings.data) == 1
|
||||
assert len(poolings.data[0].data) == len(input_tokens)
|
||||
assert poolings.usage.completion_tokens == 0
|
||||
assert poolings.usage.prompt_tokens == len(input_tokens)
|
||||
assert poolings.usage.total_tokens == len(input_tokens)
|
||||
|
||||
# test input: list[int]
|
||||
response = requests.post(
|
||||
server.url_for("pooling"),
|
||||
json={"model": model_name, "input": input_tokens, "encoding_format": "float"},
|
||||
)
|
||||
response.raise_for_status()
|
||||
poolings = PoolingResponse.model_validate(response.json())
|
||||
|
||||
assert poolings.id is not None
|
||||
assert len(poolings.data) == 1
|
||||
assert len(poolings.data[0].data) == len(input_tokens)
|
||||
assert poolings.usage.completion_tokens == 0
|
||||
assert poolings.usage.prompt_tokens == len(input_tokens)
|
||||
assert poolings.usage.total_tokens == len(input_tokens)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
def test_completion_request_batched(server: RemoteOpenAIServer, model_name: str):
|
||||
N = 10
|
||||
input_texts = [input_text] * N
|
||||
|
||||
response = requests.post(
|
||||
server.url_for("pooling"),
|
||||
json={"model": model_name, "input": input_texts, "encoding_format": "float"},
|
||||
)
|
||||
response.raise_for_status()
|
||||
poolings = PoolingResponse.model_validate(response.json())
|
||||
|
||||
assert poolings.id is not None
|
||||
assert len(poolings.data) == N
|
||||
assert len(poolings.data[0].data) == len(input_tokens)
|
||||
assert poolings.usage.completion_tokens == 0
|
||||
assert poolings.usage.prompt_tokens == len(input_tokens) * N
|
||||
assert poolings.usage.total_tokens == len(input_tokens) * N
|
||||
|
||||
# test list[list[int]]
|
||||
response = requests.post(
|
||||
server.url_for("pooling"),
|
||||
json={
|
||||
"model": model_name,
|
||||
"input": [input_tokens] * N,
|
||||
"encoding_format": "float",
|
||||
},
|
||||
)
|
||||
response.raise_for_status()
|
||||
poolings = PoolingResponse.model_validate(response.json())
|
||||
|
||||
assert poolings.id is not None
|
||||
assert len(poolings.data) == N
|
||||
assert len(poolings.data[0].data) == len(input_tokens)
|
||||
assert poolings.usage.completion_tokens == 0
|
||||
assert poolings.usage.prompt_tokens == len(input_tokens) * N
|
||||
assert poolings.usage.total_tokens == len(input_tokens) * N
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
async def test_chat_request(server: RemoteOpenAIServer, model_name: str):
|
||||
messages: list[ChatCompletionMessageParam] = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "The cat sat on the mat.",
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "A feline was resting on a rug.",
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Stars twinkle brightly in the night sky.",
|
||||
},
|
||||
]
|
||||
|
||||
# test chat request basic usage
|
||||
chat_response = requests.post(
|
||||
server.url_for("pooling"),
|
||||
json={
|
||||
"model": model_name,
|
||||
"messages": messages,
|
||||
"encoding_format": "float",
|
||||
},
|
||||
)
|
||||
chat_response.raise_for_status()
|
||||
chat_poolings = PoolingResponse.model_validate(chat_response.json())
|
||||
|
||||
tokenizer = get_tokenizer(tokenizer_name=model_name, trust_remote_code=True)
|
||||
prompt = tokenizer.apply_chat_template(
|
||||
messages,
|
||||
chat_template=DUMMY_CHAT_TEMPLATE,
|
||||
add_generation_prompt=True,
|
||||
continue_final_message=False,
|
||||
tokenize=False,
|
||||
)
|
||||
completions_response = requests.post(
|
||||
server.url_for("pooling"),
|
||||
json={
|
||||
"model": model_name,
|
||||
"input": prompt,
|
||||
"encoding_format": "float",
|
||||
# To be consistent with chat
|
||||
"add_special_tokens": False,
|
||||
},
|
||||
)
|
||||
completions_response.raise_for_status()
|
||||
completion_poolings = PoolingResponse.model_validate(completions_response.json())
|
||||
|
||||
assert chat_poolings.id is not None
|
||||
assert completion_poolings.id is not None
|
||||
assert chat_poolings.created <= completion_poolings.created
|
||||
assert chat_poolings.model_dump(exclude={"id", "created"}) == (
|
||||
completion_poolings.model_dump(exclude={"id", "created"})
|
||||
)
|
||||
|
||||
# test add_generation_prompt
|
||||
response = requests.post(
|
||||
server.url_for("pooling"),
|
||||
json={"model": model_name, "messages": messages, "add_generation_prompt": True},
|
||||
)
|
||||
|
||||
response.raise_for_status()
|
||||
output = PoolingResponse.model_validate(response.json())
|
||||
|
||||
assert output.object == "list"
|
||||
assert len(output.data) == 1
|
||||
assert output.model == MODEL_NAME
|
||||
assert output.usage.prompt_tokens == 33
|
||||
|
||||
# test continue_final_message
|
||||
# The continue_final_message parameter doesn't seem to be working with this model.
|
||||
response = requests.post(
|
||||
server.url_for("pooling"),
|
||||
json={
|
||||
"model": model_name,
|
||||
"messages": messages,
|
||||
"continue_final_message": True,
|
||||
},
|
||||
)
|
||||
|
||||
response.raise_for_status()
|
||||
output = PoolingResponse.model_validate(response.json())
|
||||
|
||||
assert output.object == "list"
|
||||
assert len(output.data) == 1
|
||||
assert output.model == MODEL_NAME
|
||||
assert output.usage.prompt_tokens == 33
|
||||
|
||||
# test add_special_tokens
|
||||
response = requests.post(
|
||||
server.url_for("pooling"),
|
||||
json={"model": model_name, "messages": messages, "add_special_tokens": True},
|
||||
)
|
||||
|
||||
response.raise_for_status()
|
||||
output = PoolingResponse.model_validate(response.json())
|
||||
|
||||
assert output.object == "list"
|
||||
assert len(output.data) == 1
|
||||
assert output.model == MODEL_NAME
|
||||
assert output.usage.prompt_tokens == 34
|
||||
|
||||
# test continue_final_message with add_generation_prompt
|
||||
response = requests.post(
|
||||
server.url_for("pooling"),
|
||||
json={
|
||||
"model": model_name,
|
||||
"messages": messages,
|
||||
"continue_final_message": True,
|
||||
"add_generation_prompt": True,
|
||||
},
|
||||
)
|
||||
assert (
|
||||
"Cannot set both `continue_final_message` and `add_generation_prompt` to True."
|
||||
in response.json()["error"]["message"]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
async def test_batch_base64_pooling(server: RemoteOpenAIServer, model_name: str):
|
||||
input_texts = [
|
||||
"Hello my name is",
|
||||
"The best thing about vLLM is that it supports many different models",
|
||||
]
|
||||
|
||||
float_response = requests.post(
|
||||
server.url_for("pooling"),
|
||||
json={
|
||||
"input": input_texts,
|
||||
"model": model_name,
|
||||
"encoding_format": "float",
|
||||
},
|
||||
)
|
||||
float_response.raise_for_status()
|
||||
responses_float = PoolingResponse.model_validate(float_response.json())
|
||||
float_data = [np.array(d.data).squeeze(-1).tolist() for d in responses_float.data]
|
||||
|
||||
base64_response = requests.post(
|
||||
server.url_for("pooling"),
|
||||
json={
|
||||
"input": input_texts,
|
||||
"model": model_name,
|
||||
"encoding_format": "base64",
|
||||
},
|
||||
)
|
||||
base64_response.raise_for_status()
|
||||
responses_base64 = PoolingResponse.model_validate(base64_response.json())
|
||||
|
||||
decoded_responses_base64_data = []
|
||||
for data in responses_base64.data:
|
||||
decoded_responses_base64_data.append(
|
||||
np.frombuffer(base64.b64decode(data.data), dtype="float32").tolist()
|
||||
)
|
||||
|
||||
check_embeddings_close(
|
||||
embeddings_0_lst=float_data,
|
||||
embeddings_1_lst=decoded_responses_base64_data,
|
||||
name_0="float32",
|
||||
name_1="base64",
|
||||
)
|
||||
|
||||
# Default response is float32 decoded from base64 by OpenAI Client
|
||||
default_response = requests.post(
|
||||
server.url_for("pooling"),
|
||||
json={
|
||||
"input": input_texts,
|
||||
"model": model_name,
|
||||
},
|
||||
)
|
||||
default_response.raise_for_status()
|
||||
responses_default = PoolingResponse.model_validate(default_response.json())
|
||||
default_data = [
|
||||
np.array(d.data).squeeze(-1).tolist() for d in responses_default.data
|
||||
]
|
||||
|
||||
check_embeddings_close(
|
||||
embeddings_0_lst=float_data,
|
||||
embeddings_1_lst=default_data,
|
||||
name_0="float32",
|
||||
name_1="default",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
async def test_base64_embed_dtype_and_endianness(
|
||||
server: RemoteOpenAIServer, model_name: str
|
||||
):
|
||||
input_texts = [input_text] * 3
|
||||
|
||||
url = server.url_for("pooling")
|
||||
float_response = requests.post(
|
||||
url,
|
||||
json={
|
||||
"model": model_name,
|
||||
"input": input_texts,
|
||||
"encoding_format": "float",
|
||||
},
|
||||
)
|
||||
responses_float = PoolingResponse.model_validate(float_response.json())
|
||||
float_data = [np.array(d.data).squeeze(-1).tolist() for d in responses_float.data]
|
||||
|
||||
for embed_dtype in EMBED_DTYPES:
|
||||
for endianness in ENDIANNESS:
|
||||
responses_base64 = requests.post(
|
||||
url,
|
||||
json={
|
||||
"model": model_name,
|
||||
"input": input_texts,
|
||||
"encoding_format": "base64",
|
||||
"embed_dtype": embed_dtype,
|
||||
"endianness": endianness,
|
||||
},
|
||||
)
|
||||
|
||||
base64_data = []
|
||||
for data in responses_base64.json()["data"]:
|
||||
binary = base64.b64decode(data["data"])
|
||||
tensor = binary2tensor(binary, (-1,), embed_dtype, endianness)
|
||||
base64_data.append(tensor.to(torch.float32).tolist())
|
||||
|
||||
check_embeddings_close(
|
||||
embeddings_0_lst=float_data,
|
||||
embeddings_1_lst=base64_data,
|
||||
name_0="float_data",
|
||||
name_1="base64_data",
|
||||
tol=1e-2,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
async def test_bytes_embed_dtype_and_endianness(
|
||||
server: RemoteOpenAIServer, model_name: str
|
||||
):
|
||||
input_texts = [input_text] * 3
|
||||
|
||||
url = server.url_for("pooling")
|
||||
float_response = requests.post(
|
||||
url,
|
||||
json={
|
||||
"model": model_name,
|
||||
"input": input_texts,
|
||||
"encoding_format": "float",
|
||||
},
|
||||
)
|
||||
responses_float = PoolingResponse.model_validate(float_response.json())
|
||||
float_data = [np.array(d.data).squeeze(-1).tolist() for d in responses_float.data]
|
||||
|
||||
for embed_dtype in EMBED_DTYPES:
|
||||
for endianness in ENDIANNESS:
|
||||
responses_bytes = requests.post(
|
||||
url,
|
||||
json={
|
||||
"model": model_name,
|
||||
"input": input_texts,
|
||||
"encoding_format": "bytes",
|
||||
"embed_dtype": embed_dtype,
|
||||
"endianness": endianness,
|
||||
},
|
||||
)
|
||||
|
||||
metadata = json.loads(responses_bytes.headers["metadata"])
|
||||
body = responses_bytes.content
|
||||
items = [MetadataItem(**x) for x in metadata["data"]]
|
||||
|
||||
bytes_data = decode_pooling_output(items=items, body=body)
|
||||
bytes_data = [x.to(torch.float32).view(-1).tolist() for x in bytes_data]
|
||||
|
||||
check_embeddings_close(
|
||||
embeddings_0_lst=float_data,
|
||||
embeddings_1_lst=bytes_data,
|
||||
name_0="float_data",
|
||||
name_1="bytes_data",
|
||||
tol=1e-2,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
async def test_bytes_only_embed_dtype_and_endianness(
|
||||
server: RemoteOpenAIServer, model_name: str
|
||||
):
|
||||
input_texts = [input_text] * 3
|
||||
|
||||
url = server.url_for("pooling")
|
||||
float_response = requests.post(
|
||||
url,
|
||||
json={
|
||||
"model": model_name,
|
||||
"input": input_texts,
|
||||
"encoding_format": "float",
|
||||
},
|
||||
)
|
||||
responses_float = PoolingResponse.model_validate(float_response.json())
|
||||
float_data = [np.array(d.data).squeeze(-1).tolist() for d in responses_float.data]
|
||||
n_tokens = responses_float.usage.prompt_tokens // len(input_texts)
|
||||
|
||||
for embed_dtype in EMBED_DTYPES:
|
||||
for endianness in ENDIANNESS:
|
||||
responses_bytes = requests.post(
|
||||
url,
|
||||
json={
|
||||
"model": model_name,
|
||||
"input": input_texts,
|
||||
"encoding_format": "bytes_only",
|
||||
"embed_dtype": embed_dtype,
|
||||
"endianness": endianness,
|
||||
},
|
||||
)
|
||||
|
||||
assert "metadata" not in responses_bytes.headers
|
||||
body = responses_bytes.content
|
||||
items = build_metadata_items(
|
||||
embed_dtype=embed_dtype,
|
||||
endianness=endianness,
|
||||
shape=(n_tokens, 1),
|
||||
n_request=len(input_texts),
|
||||
)
|
||||
bytes_data = decode_pooling_output(items=items, body=body)
|
||||
bytes_data = [x.to(torch.float32).view(-1).tolist() for x in bytes_data]
|
||||
|
||||
check_embeddings_close(
|
||||
embeddings_0_lst=float_data,
|
||||
embeddings_1_lst=bytes_data,
|
||||
name_0="float_data",
|
||||
name_1="bytes_data",
|
||||
tol=1e-2,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
@pytest.mark.parametrize("param_name", ["encoding_format", "embed_dtype", "endianness"])
|
||||
async def test_params_not_supported(
|
||||
server: RemoteOpenAIServer, model_name: str, param_name: str
|
||||
):
|
||||
responses_base64 = requests.post(
|
||||
server.url_for("pooling"),
|
||||
json={
|
||||
"model": model_name,
|
||||
"input": input_text,
|
||||
"encoding_format": "base64",
|
||||
param_name: f"bad_{param_name}",
|
||||
},
|
||||
)
|
||||
|
||||
assert responses_base64.status_code == 400
|
||||
assert "literal_error" in responses_base64.json()["error"]["message"]
|
||||
assert f"bad_{param_name}" in responses_base64.json()["error"]["message"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invocations_chat_request(server: RemoteOpenAIServer):
|
||||
request_args = {
|
||||
"model": MODEL_NAME,
|
||||
"input": input_text,
|
||||
"encoding_format": "float",
|
||||
}
|
||||
|
||||
completion_response = requests.post(server.url_for("pooling"), json=request_args)
|
||||
completion_response.raise_for_status()
|
||||
|
||||
invocation_response = requests.post(
|
||||
server.url_for("invocations"), json=request_args
|
||||
)
|
||||
invocation_response.raise_for_status()
|
||||
|
||||
completion_output = completion_response.json()
|
||||
invocation_output = invocation_response.json()
|
||||
|
||||
assert completion_output.keys() == invocation_output.keys()
|
||||
for completion_data, invocation_data in zip(
|
||||
completion_output["data"], invocation_output["data"]
|
||||
):
|
||||
assert completion_data.keys() == invocation_data.keys()
|
||||
check_embeddings_close(
|
||||
embeddings_0_lst=completion_data["data"],
|
||||
embeddings_1_lst=invocation_data["data"],
|
||||
name_0="completion",
|
||||
name_1="invocation",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invocations_conversation_chat_request(server: RemoteOpenAIServer):
|
||||
messages: list[ChatCompletionMessageParam] = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "The cat sat on the mat.",
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "A feline was resting on a rug.",
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Stars twinkle brightly in the night sky.",
|
||||
},
|
||||
]
|
||||
|
||||
request_args = {
|
||||
"model": MODEL_NAME,
|
||||
"messages": messages,
|
||||
"encoding_format": "float",
|
||||
}
|
||||
|
||||
chat_response = requests.post(server.url_for("pooling"), json=request_args)
|
||||
chat_response.raise_for_status()
|
||||
|
||||
invocation_response = requests.post(
|
||||
server.url_for("invocations"), json=request_args
|
||||
)
|
||||
invocation_response.raise_for_status()
|
||||
|
||||
chat_output = chat_response.json()
|
||||
invocation_output = invocation_response.json()
|
||||
|
||||
assert chat_output.keys() == invocation_output.keys()
|
||||
for chat_data, invocation_data in zip(
|
||||
chat_output["data"], invocation_output["data"]
|
||||
):
|
||||
assert chat_data.keys() == invocation_data.keys()
|
||||
check_embeddings_close(
|
||||
embeddings_0_lst=chat_data["data"],
|
||||
embeddings_1_lst=invocation_data["data"],
|
||||
name_0="chat",
|
||||
name_1="invocation",
|
||||
)
|
||||
@@ -1,101 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import weakref
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.entrypoints.pooling.scoring.util import EncoderScoringHfRunner
|
||||
|
||||
MODEL_NAME = "intfloat/multilingual-e5-small"
|
||||
PROMPT = "The chef prepared a delicious meal."
|
||||
EMBEDDING_SIZE = 384
|
||||
|
||||
TEXTS_1 = [
|
||||
"What is the capital of France?",
|
||||
"What is the capital of Germany?",
|
||||
]
|
||||
|
||||
TEXTS_2 = [
|
||||
"The capital of France is Paris.",
|
||||
"The capital of Germany is Berlin.",
|
||||
]
|
||||
|
||||
DTYPE = "half"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def llm(vllm_runner):
|
||||
with vllm_runner(
|
||||
MODEL_NAME,
|
||||
max_model_len=None,
|
||||
max_num_batched_tokens=32768,
|
||||
tensor_parallel_size=1,
|
||||
gpu_memory_utilization=0.75,
|
||||
enforce_eager=True,
|
||||
seed=0,
|
||||
enable_chunked_prefill=None,
|
||||
) as runner:
|
||||
# pytest caches yielded fixtures until after teardown, so use a proxy to
|
||||
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
|
||||
yield weakref.proxy(runner.llm)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def hf_model():
|
||||
return EncoderScoringHfRunner(MODEL_NAME)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_1_to_1(llm, hf_model):
|
||||
text_pair = [TEXTS_1[0], TEXTS_2[0]]
|
||||
|
||||
hf_outputs = hf_model.predict([text_pair]).tolist()
|
||||
vllm_outputs = [
|
||||
output.outputs.score for output in llm.score(text_pair[0], text_pair[1])
|
||||
]
|
||||
|
||||
assert len(vllm_outputs) == 1
|
||||
assert len(hf_outputs) == 1
|
||||
|
||||
assert hf_outputs[0] == pytest.approx(vllm_outputs[0], rel=0.01)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_1_to_n(llm, hf_model):
|
||||
text_pairs = [
|
||||
[TEXTS_1[0], TEXTS_2[0]],
|
||||
[TEXTS_1[0], TEXTS_2[1]],
|
||||
]
|
||||
|
||||
hf_outputs = hf_model.predict(text_pairs).tolist()
|
||||
vllm_outputs = [output.outputs.score for output in llm.score(TEXTS_1[0], TEXTS_2)]
|
||||
|
||||
assert len(vllm_outputs) == 2
|
||||
assert len(hf_outputs) == 2
|
||||
|
||||
assert hf_outputs[0] == pytest.approx(vllm_outputs[0], rel=0.01)
|
||||
assert hf_outputs[1] == pytest.approx(vllm_outputs[1], rel=0.01)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_n_to_n(llm, hf_model):
|
||||
text_pairs = [
|
||||
[TEXTS_1[0], TEXTS_2[0]],
|
||||
[TEXTS_1[1], TEXTS_2[1]],
|
||||
]
|
||||
|
||||
hf_outputs = hf_model.predict(text_pairs).tolist()
|
||||
vllm_outputs = [output.outputs.score for output in llm.score(TEXTS_1, TEXTS_2)]
|
||||
|
||||
assert len(vllm_outputs) == 2
|
||||
assert len(hf_outputs) == 2
|
||||
|
||||
assert hf_outputs[0] == pytest.approx(vllm_outputs[0], rel=0.01)
|
||||
assert hf_outputs[1] == pytest.approx(vllm_outputs[1], rel=0.01)
|
||||
|
||||
|
||||
def test_embed(llm):
|
||||
outputs = llm.encode(PROMPT, pooling_task="embed", use_tqdm=False)
|
||||
assert len(outputs) == 1
|
||||
assert len(outputs[0].outputs.data) == EMBEDDING_SIZE
|
||||
@@ -1,384 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from tests.entrypoints.pooling.scoring.util import EncoderScoringHfRunner
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.entrypoints.pooling.pooling.protocol import PoolingResponse
|
||||
from vllm.entrypoints.pooling.scoring.protocol import RerankResponse, ScoreResponse
|
||||
|
||||
MODEL_NAME = "BAAI/bge-base-en-v1.5"
|
||||
input_text = "This product was excellent and exceeded my expectations"
|
||||
DTYPE = "half"
|
||||
EMBEDDING_SIZE = 768
|
||||
|
||||
TEXTS_1 = [
|
||||
"What is the capital of France?",
|
||||
"What is the capital of Germany?",
|
||||
]
|
||||
|
||||
TEXTS_2 = [
|
||||
"The capital of France is Paris.",
|
||||
"The capital of Germany is Berlin.",
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
args = ["--enforce-eager", "--max-model-len", "100", "--dtype", DTYPE]
|
||||
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def hf_model():
|
||||
return EncoderScoringHfRunner(MODEL_NAME)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_score_api_queries_str_1_documents_str_1(
|
||||
hf_model, server: RemoteOpenAIServer
|
||||
):
|
||||
score_response = requests.post(
|
||||
server.url_for("score"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"queries": TEXTS_1[0],
|
||||
"documents": TEXTS_2[0],
|
||||
},
|
||||
)
|
||||
score_response.raise_for_status()
|
||||
score = ScoreResponse.model_validate(score_response.json())
|
||||
|
||||
assert score.id is not None
|
||||
assert score.data is not None
|
||||
assert len(score.data) == 1
|
||||
|
||||
vllm_outputs = [d.score for d in score.data]
|
||||
hf_outputs = hf_model.predict([[TEXTS_1[0], TEXTS_2[0]]]).tolist()
|
||||
|
||||
for i in range(len(vllm_outputs)):
|
||||
assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_score_api_queries_str_1_documents_str_n(
|
||||
hf_model, server: RemoteOpenAIServer
|
||||
):
|
||||
text_pairs = [
|
||||
[TEXTS_1[0], TEXTS_2[0]],
|
||||
[TEXTS_1[0], TEXTS_2[1]],
|
||||
]
|
||||
|
||||
score_response = requests.post(
|
||||
server.url_for("score"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"queries": TEXTS_1[0],
|
||||
"documents": TEXTS_2,
|
||||
},
|
||||
)
|
||||
score_response.raise_for_status()
|
||||
score = ScoreResponse.model_validate(score_response.json())
|
||||
|
||||
assert score.id is not None
|
||||
assert score.data is not None
|
||||
assert len(score.data) == 2
|
||||
|
||||
vllm_outputs = [d.score for d in score.data]
|
||||
hf_outputs = hf_model.predict(text_pairs).tolist()
|
||||
|
||||
for i in range(len(vllm_outputs)):
|
||||
assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_score_api_queries_str_n_documents_str_n(
|
||||
hf_model, server: RemoteOpenAIServer
|
||||
):
|
||||
text_pairs = [
|
||||
[TEXTS_1[0], TEXTS_2[0]],
|
||||
[TEXTS_1[1], TEXTS_2[1]],
|
||||
]
|
||||
|
||||
score_response = requests.post(
|
||||
server.url_for("score"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"queries": TEXTS_1,
|
||||
"documents": TEXTS_2,
|
||||
},
|
||||
)
|
||||
score_response.raise_for_status()
|
||||
score = ScoreResponse.model_validate(score_response.json())
|
||||
|
||||
assert score.id is not None
|
||||
assert score.data is not None
|
||||
assert len(score.data) == 2
|
||||
|
||||
vllm_outputs = [d.score for d in score.data]
|
||||
hf_outputs = hf_model.predict(text_pairs).tolist()
|
||||
|
||||
for i in range(len(vllm_outputs)):
|
||||
assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_score_api_queries_vs_items(hf_model, server: RemoteOpenAIServer):
|
||||
text_pairs = [
|
||||
[TEXTS_1[0], TEXTS_2[0]],
|
||||
[TEXTS_1[1], TEXTS_2[1]],
|
||||
]
|
||||
|
||||
score_response = requests.post(
|
||||
server.url_for("score"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"queries": TEXTS_1,
|
||||
"items": TEXTS_2,
|
||||
},
|
||||
)
|
||||
score_response.raise_for_status()
|
||||
score = ScoreResponse.model_validate(score_response.json())
|
||||
|
||||
assert score.id is not None
|
||||
assert score.data is not None
|
||||
assert len(score.data) == 2
|
||||
|
||||
vllm_outputs = [d.score for d in score.data]
|
||||
hf_outputs = hf_model.predict(text_pairs).tolist()
|
||||
|
||||
for i in range(len(vllm_outputs)):
|
||||
assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_score_api_text_1_vs_text_2(hf_model, server: RemoteOpenAIServer):
|
||||
text_pairs = [
|
||||
[TEXTS_1[0], TEXTS_2[0]],
|
||||
[TEXTS_1[1], TEXTS_2[1]],
|
||||
]
|
||||
|
||||
score_response = requests.post(
|
||||
server.url_for("score"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"text_1": TEXTS_1,
|
||||
"text_2": TEXTS_2,
|
||||
},
|
||||
)
|
||||
score_response.raise_for_status()
|
||||
score = ScoreResponse.model_validate(score_response.json())
|
||||
|
||||
assert score.id is not None
|
||||
assert score.data is not None
|
||||
assert len(score.data) == 2
|
||||
|
||||
vllm_outputs = [d.score for d in score.data]
|
||||
hf_outputs = hf_model.predict(text_pairs).tolist()
|
||||
|
||||
for i in range(len(vllm_outputs)):
|
||||
assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_score_api_data_1_vs_data_2(hf_model, server: RemoteOpenAIServer):
|
||||
text_pairs = [
|
||||
[TEXTS_1[0], TEXTS_2[0]],
|
||||
[TEXTS_1[1], TEXTS_2[1]],
|
||||
]
|
||||
|
||||
score_response = requests.post(
|
||||
server.url_for("score"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"data_1": TEXTS_1,
|
||||
"data_2": TEXTS_2,
|
||||
},
|
||||
)
|
||||
score_response.raise_for_status()
|
||||
score = ScoreResponse.model_validate(score_response.json())
|
||||
|
||||
assert score.id is not None
|
||||
assert score.data is not None
|
||||
assert len(score.data) == 2
|
||||
|
||||
vllm_outputs = [d.score for d in score.data]
|
||||
hf_outputs = hf_model.predict(text_pairs).tolist()
|
||||
|
||||
for i in range(len(vllm_outputs)):
|
||||
assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rerank_api_texts(server: RemoteOpenAIServer):
|
||||
query = "What is the capital of France?"
|
||||
documents = [
|
||||
"The capital of Brazil is Brasilia.",
|
||||
"The capital of France is Paris.",
|
||||
]
|
||||
|
||||
rerank_response = requests.post(
|
||||
server.url_for("rerank"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"query": query,
|
||||
"documents": documents,
|
||||
},
|
||||
)
|
||||
rerank_response.raise_for_status()
|
||||
rerank = RerankResponse.model_validate(rerank_response.json())
|
||||
|
||||
assert rerank.id is not None
|
||||
assert rerank.results is not None
|
||||
assert len(rerank.results) == 2
|
||||
paris_result = next(r for r in rerank.results if r.index == 1)
|
||||
brazil_result = next(r for r in rerank.results if r.index == 0)
|
||||
assert paris_result.relevance_score > brazil_result.relevance_score
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rerank_api_top_n(server: RemoteOpenAIServer):
|
||||
query = "What is the capital of France?"
|
||||
documents = [
|
||||
"The capital of Brazil is Brasilia.",
|
||||
"The capital of France is Paris.",
|
||||
"Cross-encoder models are neat",
|
||||
]
|
||||
|
||||
rerank_response = requests.post(
|
||||
server.url_for("rerank"),
|
||||
json={"model": MODEL_NAME, "query": query, "documents": documents, "top_n": 2},
|
||||
)
|
||||
rerank_response.raise_for_status()
|
||||
rerank = RerankResponse.model_validate(rerank_response.json())
|
||||
|
||||
assert rerank.id is not None
|
||||
assert rerank.results is not None
|
||||
assert len(rerank.results) == 2
|
||||
assert rerank.results[0].index == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rerank_api_max_model_len(server: RemoteOpenAIServer):
|
||||
query = "What is the capital of France?" * 100
|
||||
documents = [
|
||||
"The capital of Brazil is Brasilia.",
|
||||
"The capital of France is Paris.",
|
||||
]
|
||||
|
||||
rerank_response = requests.post(
|
||||
server.url_for("rerank"),
|
||||
json={"model": MODEL_NAME, "query": query, "documents": documents},
|
||||
)
|
||||
assert rerank_response.status_code == 400
|
||||
# Assert just a small fragments of the response
|
||||
assert "Please reduce the length of the input prompt" in rerank_response.text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_score_api_max_model_len(server: RemoteOpenAIServer):
|
||||
queries = "What is the capital of France?" * 20
|
||||
documents = [
|
||||
"The capital of Brazil is Brasilia.",
|
||||
"The capital of France is Paris.",
|
||||
]
|
||||
|
||||
score_response = requests.post(
|
||||
server.url_for("score"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"queries": queries,
|
||||
"documents": documents,
|
||||
},
|
||||
)
|
||||
assert score_response.status_code == 400
|
||||
# Assert just a small fragments of the response
|
||||
assert "Please reduce the length of the input prompt" in score_response.text
|
||||
|
||||
# Test truncation
|
||||
score_response = requests.post(
|
||||
server.url_for("score"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"queries": queries,
|
||||
"documents": documents,
|
||||
"truncate_prompt_tokens": 101,
|
||||
},
|
||||
)
|
||||
assert score_response.status_code == 400
|
||||
assert "Please request a smaller truncation size." in score_response.text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invocations(server: RemoteOpenAIServer):
|
||||
query = "What is the capital of France?"
|
||||
documents = [
|
||||
"The capital of Brazil is Brasilia.",
|
||||
"The capital of France is Paris.",
|
||||
]
|
||||
|
||||
request_args = {
|
||||
"model": MODEL_NAME,
|
||||
"query": query,
|
||||
"documents": documents,
|
||||
}
|
||||
|
||||
rerank_response = requests.post(server.url_for("rerank"), json=request_args)
|
||||
rerank_response.raise_for_status()
|
||||
|
||||
invocation_response = requests.post(
|
||||
server.url_for("invocations"), json=request_args
|
||||
)
|
||||
invocation_response.raise_for_status()
|
||||
|
||||
rerank_output = rerank_response.json()
|
||||
invocation_output = invocation_response.json()
|
||||
|
||||
assert rerank_output.keys() == invocation_output.keys()
|
||||
for rerank_result, invocations_result in zip(
|
||||
rerank_output["results"], invocation_output["results"]
|
||||
):
|
||||
assert rerank_result.keys() == invocations_result.keys()
|
||||
assert rerank_result["relevance_score"] == pytest.approx(
|
||||
invocations_result["relevance_score"], rel=0.01
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pooling_embed(server: RemoteOpenAIServer):
|
||||
response = requests.post(
|
||||
server.url_for("pooling"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"input": input_text,
|
||||
"encoding_format": "float",
|
||||
"task": "embed",
|
||||
},
|
||||
)
|
||||
poolings = PoolingResponse.model_validate(response.json())
|
||||
assert len(poolings.data) == 1
|
||||
assert len(poolings.data[0].data) == EMBEDDING_SIZE
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("task", ["classify", "token_classify", "plugin"])
|
||||
async def test_pooling_not_supported(server: RemoteOpenAIServer, task: str):
|
||||
response = requests.post(
|
||||
server.url_for("pooling"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"input": input_text,
|
||||
"encoding_format": "float",
|
||||
"task": task,
|
||||
},
|
||||
)
|
||||
assert response.json()["error"]["type"] == "BadRequestError"
|
||||
if task == "plugin":
|
||||
err_msg = "No IOProcessor plugin installed."
|
||||
else:
|
||||
err_msg = f"Unsupported task: {task!r}"
|
||||
assert response.json()["error"]["message"].startswith(err_msg)
|
||||
@@ -1,56 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.models.language.pooling_mteb_test.mteb_score_utils import (
|
||||
MTEB_RERANK_LANGS,
|
||||
MTEB_RERANK_TASKS,
|
||||
MTEB_RERANK_TOL,
|
||||
RerankClientMtebEncoder,
|
||||
ScoreClientMtebEncoder,
|
||||
run_mteb_rerank,
|
||||
)
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
|
||||
os.environ["VLLM_LOGGING_LEVEL"] = "WARNING"
|
||||
|
||||
MODEL_NAME = "cross-encoder/ms-marco-MiniLM-L-6-v2"
|
||||
st_main_score = 0.33457
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
args = ["--runner", "pooling", "--enforce-eager", "--disable-uvicorn-access-log"]
|
||||
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
def test_mteb_score(server):
|
||||
url = server.url_for("score")
|
||||
encoder = ScoreClientMtebEncoder(MODEL_NAME, url)
|
||||
vllm_main_score = run_mteb_rerank(encoder, MTEB_RERANK_TASKS, MTEB_RERANK_LANGS)
|
||||
|
||||
print("VLLM main score: ", vllm_main_score)
|
||||
print("SentenceTransformer main score: ", st_main_score)
|
||||
print("Difference: ", st_main_score - vllm_main_score)
|
||||
|
||||
# We are not concerned that the vllm mteb results are better
|
||||
# than SentenceTransformers, so we only perform one-sided testing.
|
||||
assert st_main_score - vllm_main_score < MTEB_RERANK_TOL
|
||||
|
||||
|
||||
def test_mteb_rerank(server):
|
||||
url = server.url_for("rerank")
|
||||
encoder = RerankClientMtebEncoder(MODEL_NAME, url)
|
||||
vllm_main_score = run_mteb_rerank(encoder, MTEB_RERANK_TASKS, MTEB_RERANK_LANGS)
|
||||
|
||||
print("VLLM main score: ", vllm_main_score)
|
||||
print("SentenceTransformer main score: ", st_main_score)
|
||||
print("Difference: ", st_main_score - vllm_main_score)
|
||||
|
||||
# We are not concerned that the vllm mteb results are better
|
||||
# than SentenceTransformers, so we only perform one-sided testing.
|
||||
assert st_main_score - vllm_main_score < MTEB_RERANK_TOL
|
||||
@@ -1,154 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import weakref
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.models.utils import softmax
|
||||
from vllm import LLM, PoolingParams
|
||||
|
||||
MODEL_NAME = "tomaarsen/Qwen3-Reranker-0.6B-seq-cls"
|
||||
PROMPT = "The chef prepared a delicious meal."
|
||||
TEXTS_1 = [
|
||||
"What is the capital of France?",
|
||||
"What is the capital of Germany?",
|
||||
]
|
||||
|
||||
TEXTS_2 = [
|
||||
"The capital of France is Paris.",
|
||||
"The capital of Germany is Berlin.",
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def llm(vllm_runner):
|
||||
with vllm_runner(
|
||||
MODEL_NAME,
|
||||
max_model_len=None,
|
||||
max_num_batched_tokens=32768,
|
||||
tensor_parallel_size=1,
|
||||
gpu_memory_utilization=0.75,
|
||||
enforce_eager=True,
|
||||
seed=0,
|
||||
enable_chunked_prefill=None,
|
||||
) as runner:
|
||||
# pytest caches yielded fixtures until after teardown, so use a proxy to
|
||||
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
|
||||
yield weakref.proxy(runner.llm)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def hf_model(hf_runner):
|
||||
return hf_runner(MODEL_NAME, is_cross_encoder=True)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_1_to_1(llm, hf_model):
|
||||
text_pair = [TEXTS_1[0], TEXTS_2[0]]
|
||||
|
||||
hf_outputs = hf_model.predict([text_pair]).tolist()
|
||||
vllm_outputs = [
|
||||
output.outputs.score for output in llm.score(text_pair[0], text_pair[1])
|
||||
]
|
||||
|
||||
assert len(vllm_outputs) == 1
|
||||
assert len(hf_outputs) == 1
|
||||
|
||||
assert hf_outputs[0] == pytest.approx(vllm_outputs[0], rel=0.01)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_1_to_n(llm, hf_model):
|
||||
text_pairs = [
|
||||
[TEXTS_1[0], TEXTS_2[0]],
|
||||
[TEXTS_1[0], TEXTS_2[1]],
|
||||
]
|
||||
|
||||
vllm_outputs = [output.outputs.score for output in llm.score(TEXTS_1[0], TEXTS_2)]
|
||||
hf_outputs = hf_model.predict(text_pairs).tolist()
|
||||
|
||||
assert len(vllm_outputs) == 2
|
||||
assert len(hf_outputs) == 2
|
||||
|
||||
assert hf_outputs[0] == pytest.approx(vllm_outputs[0], rel=0.01)
|
||||
assert hf_outputs[1] == pytest.approx(vllm_outputs[1], rel=0.01)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_n_to_n(llm, hf_model):
|
||||
text_pairs = [
|
||||
[TEXTS_1[0], TEXTS_2[0]],
|
||||
[TEXTS_1[1], TEXTS_2[1]],
|
||||
]
|
||||
|
||||
vllm_outputs = [output.outputs.score for output in llm.score(TEXTS_1, TEXTS_2)]
|
||||
hf_outputs = hf_model.predict(text_pairs).tolist()
|
||||
|
||||
assert len(vllm_outputs) == 2
|
||||
assert len(hf_outputs) == 2
|
||||
|
||||
assert hf_outputs[0] == pytest.approx(vllm_outputs[0], rel=0.01)
|
||||
assert hf_outputs[1] == pytest.approx(vllm_outputs[1], rel=0.01)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_classify(llm):
|
||||
outputs = llm.encode(PROMPT, pooling_task="classify", use_tqdm=False)
|
||||
assert len(outputs) == 1
|
||||
assert len(outputs[0].outputs.data) == 1
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_max_tokens_per_doc(llm: LLM):
|
||||
"""Test max_tokens_per_doc via PoolingParams.extra_kwargs (offline)."""
|
||||
long_doc = "The capital of France is Paris. " * 20
|
||||
|
||||
# Without truncation
|
||||
outputs_no_limit = llm.score(
|
||||
TEXTS_1[0],
|
||||
long_doc,
|
||||
use_tqdm=False,
|
||||
)
|
||||
|
||||
# With truncation via extra_kwargs
|
||||
outputs_with_limit = llm.score(
|
||||
TEXTS_1[0],
|
||||
long_doc,
|
||||
pooling_params=PoolingParams(extra_kwargs={"max_tokens_per_doc": 10}),
|
||||
use_tqdm=False,
|
||||
)
|
||||
|
||||
assert len(outputs_no_limit) == 1
|
||||
assert len(outputs_with_limit) == 1
|
||||
|
||||
# Truncated version should have fewer prompt tokens
|
||||
no_limit_tokens = len(outputs_no_limit[0].prompt_token_ids)
|
||||
with_limit_tokens = len(outputs_with_limit[0].prompt_token_ids)
|
||||
assert with_limit_tokens < no_limit_tokens
|
||||
|
||||
|
||||
def test_pooling_params(llm: LLM):
|
||||
def get_outputs(use_activation):
|
||||
outputs = llm.score(
|
||||
TEXTS_1[0],
|
||||
TEXTS_2[0],
|
||||
pooling_params=PoolingParams(use_activation=use_activation),
|
||||
use_tqdm=False,
|
||||
)
|
||||
return torch.tensor([x.outputs.score for x in outputs])
|
||||
|
||||
default = get_outputs(use_activation=None)
|
||||
w_activation = get_outputs(use_activation=True)
|
||||
wo_activation = get_outputs(use_activation=False)
|
||||
|
||||
assert torch.allclose(default, w_activation, atol=1e-2), (
|
||||
"Default should use activation."
|
||||
)
|
||||
assert not torch.allclose(w_activation, wo_activation, atol=1e-2), (
|
||||
"wo_activation should not use activation."
|
||||
)
|
||||
assert torch.allclose(softmax(wo_activation), w_activation, atol=1e-2), (
|
||||
"w_activation should be close to activation(wo_activation)."
|
||||
)
|
||||
@@ -1,512 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.entrypoints.pooling.pooling.protocol import PoolingResponse
|
||||
from vllm.entrypoints.pooling.scoring.protocol import RerankResponse, ScoreResponse
|
||||
|
||||
MODEL_NAME = "BAAI/bge-reranker-base"
|
||||
DTYPE = "half"
|
||||
input_text = "This product was excellent and exceeded my expectations"
|
||||
input_tokens = [0, 3293, 12996, 509, 40881, 136, 204839, 297, 759, 202702, 2]
|
||||
|
||||
|
||||
TEXTS_1 = [
|
||||
"What is the capital of France?",
|
||||
"What is the capital of Germany?",
|
||||
]
|
||||
|
||||
TEXTS_2 = [
|
||||
"The capital of France is Paris.",
|
||||
"The capital of Germany is Berlin.",
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
args = ["--enforce-eager", "--max-model-len", "100", "--dtype", DTYPE]
|
||||
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def hf_model(hf_runner):
|
||||
return hf_runner(MODEL_NAME, is_cross_encoder=True)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_basic(server: RemoteOpenAIServer):
|
||||
# test /v1/models
|
||||
response = requests.get(server.url_for("/v1/models"))
|
||||
served_model = response.json()["data"][0]["id"]
|
||||
assert served_model == MODEL_NAME
|
||||
|
||||
# test /tokenize
|
||||
response = requests.post(
|
||||
server.url_for("/tokenize"),
|
||||
json={"model": MODEL_NAME, "prompt": input_text},
|
||||
)
|
||||
assert response.json()["tokens"] == input_tokens
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_score_api_queries_str_1_documents_str_1(
|
||||
hf_model, server: RemoteOpenAIServer
|
||||
):
|
||||
score_response = requests.post(
|
||||
server.url_for("score"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"queries": TEXTS_1[0],
|
||||
"documents": TEXTS_2[0],
|
||||
},
|
||||
)
|
||||
score_response.raise_for_status()
|
||||
score = ScoreResponse.model_validate(score_response.json())
|
||||
|
||||
assert score.id is not None
|
||||
assert score.data is not None
|
||||
assert len(score.data) == 1
|
||||
|
||||
vllm_outputs = [d.score for d in score.data]
|
||||
hf_outputs = hf_model.predict([[TEXTS_1[0], TEXTS_2[0]]]).tolist()
|
||||
|
||||
for i in range(len(vllm_outputs)):
|
||||
assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_score_api_queries_str_1_documents_str_n(
|
||||
hf_model, server: RemoteOpenAIServer
|
||||
):
|
||||
text_pairs = [
|
||||
[TEXTS_1[0], TEXTS_2[0]],
|
||||
[TEXTS_1[0], TEXTS_2[1]],
|
||||
]
|
||||
|
||||
score_response = requests.post(
|
||||
server.url_for("score"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"queries": TEXTS_1[0],
|
||||
"documents": TEXTS_2,
|
||||
},
|
||||
)
|
||||
score_response.raise_for_status()
|
||||
score = ScoreResponse.model_validate(score_response.json())
|
||||
|
||||
assert score.id is not None
|
||||
assert score.data is not None
|
||||
assert len(score.data) == 2
|
||||
|
||||
vllm_outputs = [d.score for d in score.data]
|
||||
hf_outputs = hf_model.predict(text_pairs).tolist()
|
||||
|
||||
for i in range(len(vllm_outputs)):
|
||||
assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_score_api_queries_str_n_documents_str_n(
|
||||
hf_model, server: RemoteOpenAIServer
|
||||
):
|
||||
text_pairs = [
|
||||
[TEXTS_1[0], TEXTS_2[0]],
|
||||
[TEXTS_1[1], TEXTS_2[1]],
|
||||
]
|
||||
|
||||
score_response = requests.post(
|
||||
server.url_for("score"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"queries": TEXTS_1,
|
||||
"documents": TEXTS_2,
|
||||
},
|
||||
)
|
||||
score_response.raise_for_status()
|
||||
score = ScoreResponse.model_validate(score_response.json())
|
||||
|
||||
assert score.id is not None
|
||||
assert score.data is not None
|
||||
assert len(score.data) == 2
|
||||
|
||||
vllm_outputs = [d.score for d in score.data]
|
||||
hf_outputs = hf_model.predict(text_pairs).tolist()
|
||||
|
||||
for i in range(len(vllm_outputs)):
|
||||
assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_score_api_queries_vs_items(hf_model, server: RemoteOpenAIServer):
|
||||
text_pairs = [
|
||||
[TEXTS_1[0], TEXTS_2[0]],
|
||||
[TEXTS_1[1], TEXTS_2[1]],
|
||||
]
|
||||
|
||||
score_response = requests.post(
|
||||
server.url_for("score"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"queries": TEXTS_1,
|
||||
"items": TEXTS_2,
|
||||
},
|
||||
)
|
||||
score_response.raise_for_status()
|
||||
score = ScoreResponse.model_validate(score_response.json())
|
||||
|
||||
assert score.id is not None
|
||||
assert score.data is not None
|
||||
assert len(score.data) == 2
|
||||
|
||||
vllm_outputs = [d.score for d in score.data]
|
||||
hf_outputs = hf_model.predict(text_pairs).tolist()
|
||||
|
||||
for i in range(len(vllm_outputs)):
|
||||
assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_score_api_text_1_vs_text_2(hf_model, server: RemoteOpenAIServer):
|
||||
text_pairs = [
|
||||
[TEXTS_1[0], TEXTS_2[0]],
|
||||
[TEXTS_1[1], TEXTS_2[1]],
|
||||
]
|
||||
|
||||
score_response = requests.post(
|
||||
server.url_for("score"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"text_1": TEXTS_1,
|
||||
"text_2": TEXTS_2,
|
||||
},
|
||||
)
|
||||
score_response.raise_for_status()
|
||||
score = ScoreResponse.model_validate(score_response.json())
|
||||
|
||||
assert score.id is not None
|
||||
assert score.data is not None
|
||||
assert len(score.data) == 2
|
||||
|
||||
vllm_outputs = [d.score for d in score.data]
|
||||
hf_outputs = hf_model.predict(text_pairs).tolist()
|
||||
|
||||
for i in range(len(vllm_outputs)):
|
||||
assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_score_api_data_1_vs_data_2(hf_model, server: RemoteOpenAIServer):
|
||||
text_pairs = [
|
||||
[TEXTS_1[0], TEXTS_2[0]],
|
||||
[TEXTS_1[1], TEXTS_2[1]],
|
||||
]
|
||||
|
||||
score_response = requests.post(
|
||||
server.url_for("score"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"data_1": TEXTS_1,
|
||||
"data_2": TEXTS_2,
|
||||
},
|
||||
)
|
||||
score_response.raise_for_status()
|
||||
score = ScoreResponse.model_validate(score_response.json())
|
||||
|
||||
assert score.id is not None
|
||||
assert score.data is not None
|
||||
assert len(score.data) == 2
|
||||
|
||||
vllm_outputs = [d.score for d in score.data]
|
||||
hf_outputs = hf_model.predict(text_pairs).tolist()
|
||||
|
||||
for i in range(len(vllm_outputs)):
|
||||
assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rerank_api_texts(server: RemoteOpenAIServer):
|
||||
query = "What is the capital of France?"
|
||||
documents = [
|
||||
"The capital of Brazil is Brasilia.",
|
||||
"The capital of France is Paris.",
|
||||
]
|
||||
|
||||
rerank_response = requests.post(
|
||||
server.url_for("rerank"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"query": query,
|
||||
"documents": documents,
|
||||
},
|
||||
)
|
||||
rerank_response.raise_for_status()
|
||||
rerank = RerankResponse.model_validate(rerank_response.json())
|
||||
|
||||
assert rerank.id is not None
|
||||
assert rerank.results is not None
|
||||
assert len(rerank.results) == 2
|
||||
assert rerank.results[0].relevance_score >= 0.9
|
||||
assert rerank.results[1].relevance_score <= 0.01
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rerank_api_top_n(server: RemoteOpenAIServer):
|
||||
query = "What is the capital of France?"
|
||||
documents = [
|
||||
"The capital of Brazil is Brasilia.",
|
||||
"The capital of France is Paris.",
|
||||
"Cross-encoder models are neat",
|
||||
]
|
||||
|
||||
rerank_response = requests.post(
|
||||
server.url_for("rerank"),
|
||||
json={"model": MODEL_NAME, "query": query, "documents": documents, "top_n": 2},
|
||||
)
|
||||
rerank_response.raise_for_status()
|
||||
rerank = RerankResponse.model_validate(rerank_response.json())
|
||||
|
||||
assert rerank.id is not None
|
||||
assert rerank.results is not None
|
||||
assert len(rerank.results) == 2
|
||||
assert rerank.results[0].relevance_score >= 0.9
|
||||
assert rerank.results[1].relevance_score <= 0.01
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rerank_api_max_model_len(server: RemoteOpenAIServer):
|
||||
query = "What is the capital of France?" * 100
|
||||
documents = [
|
||||
"The capital of Brazil is Brasilia.",
|
||||
"The capital of France is Paris.",
|
||||
]
|
||||
|
||||
rerank_response = requests.post(
|
||||
server.url_for("rerank"),
|
||||
json={"model": MODEL_NAME, "query": query, "documents": documents},
|
||||
)
|
||||
assert rerank_response.status_code == 400
|
||||
# Assert just a small fragments of the response
|
||||
assert "Please reduce the length of the input prompt" in rerank_response.text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_score_api_max_model_len(server: RemoteOpenAIServer):
|
||||
queries = "What is the capital of France?" * 20
|
||||
documents = [
|
||||
"The capital of Brazil is Brasilia.",
|
||||
"The capital of France is Paris.",
|
||||
]
|
||||
|
||||
score_response = requests.post(
|
||||
server.url_for("score"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"queries": queries,
|
||||
"documents": documents,
|
||||
},
|
||||
)
|
||||
assert score_response.status_code == 400
|
||||
# Assert just a small fragments of the response
|
||||
assert "Please reduce the length of the input prompt" in score_response.text
|
||||
|
||||
# Test truncation
|
||||
score_response = requests.post(
|
||||
server.url_for("score"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"queries": queries,
|
||||
"documents": documents,
|
||||
"truncate_prompt_tokens": 101,
|
||||
},
|
||||
)
|
||||
assert score_response.status_code == 400
|
||||
assert "Please request a smaller truncation size." in score_response.text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invocations(server: RemoteOpenAIServer):
|
||||
query = "What is the capital of France?"
|
||||
documents = [
|
||||
"The capital of Brazil is Brasilia.",
|
||||
"The capital of France is Paris.",
|
||||
]
|
||||
|
||||
request_args = {
|
||||
"model": MODEL_NAME,
|
||||
"query": query,
|
||||
"documents": documents,
|
||||
}
|
||||
|
||||
rerank_response = requests.post(server.url_for("rerank"), json=request_args)
|
||||
rerank_response.raise_for_status()
|
||||
|
||||
invocation_response = requests.post(
|
||||
server.url_for("invocations"), json=request_args
|
||||
)
|
||||
invocation_response.raise_for_status()
|
||||
|
||||
rerank_output = rerank_response.json()
|
||||
invocation_output = invocation_response.json()
|
||||
|
||||
assert rerank_output.keys() == invocation_output.keys()
|
||||
for rerank_result, invocations_result in zip(
|
||||
rerank_output["results"], invocation_output["results"]
|
||||
):
|
||||
assert rerank_result.keys() == invocations_result.keys()
|
||||
assert rerank_result["relevance_score"] == pytest.approx(
|
||||
invocations_result["relevance_score"], rel=0.01
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_use_activation(server: RemoteOpenAIServer):
|
||||
async def get_outputs(use_activation):
|
||||
query = "What is the capital of France?"
|
||||
documents = [
|
||||
"The capital of Brazil is Brasilia.",
|
||||
"The capital of France is Paris.",
|
||||
]
|
||||
|
||||
response = requests.post(
|
||||
server.url_for("rerank"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"query": query,
|
||||
"documents": documents,
|
||||
"use_activation": use_activation,
|
||||
},
|
||||
)
|
||||
outputs = response.json()
|
||||
|
||||
return torch.tensor([x["relevance_score"] for x in outputs["results"]])
|
||||
|
||||
default = await get_outputs(use_activation=None)
|
||||
w_activation = await get_outputs(use_activation=True)
|
||||
wo_activation = await get_outputs(use_activation=False)
|
||||
|
||||
assert torch.allclose(default, w_activation, atol=1e-2), (
|
||||
"Default should use activation."
|
||||
)
|
||||
assert not torch.allclose(w_activation, wo_activation, atol=1e-2), (
|
||||
"wo_activation should not use activation."
|
||||
)
|
||||
assert torch.allclose(F.sigmoid(wo_activation), w_activation, atol=1e-2), (
|
||||
"w_activation should be close to activation(wo_activation)."
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pooling_classify(server: RemoteOpenAIServer):
|
||||
response = requests.post(
|
||||
server.url_for("pooling"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"input": input_text,
|
||||
"encoding_format": "float",
|
||||
"task": "classify",
|
||||
},
|
||||
)
|
||||
poolings = PoolingResponse.model_validate(response.json())
|
||||
assert len(poolings.data) == 1
|
||||
assert len(poolings.data[0].data) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rerank_max_tokens_per_doc(
|
||||
server: RemoteOpenAIServer,
|
||||
):
|
||||
"""Test that max_tokens_per_doc actually reduces the token count."""
|
||||
query = "What is the capital of France?"
|
||||
# Use a doc that fits within max_model_len=100 (query ~8 tokens + 4 special)
|
||||
long_doc = "The capital of France is Paris. " * 10 # ~70 tokens
|
||||
|
||||
# Without max_tokens_per_doc
|
||||
response_no_limit = requests.post(
|
||||
server.url_for("rerank"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"query": query,
|
||||
"documents": [long_doc],
|
||||
"truncate_prompt_tokens": 99,
|
||||
},
|
||||
)
|
||||
response_no_limit.raise_for_status()
|
||||
rerank_no_limit = RerankResponse.model_validate(response_no_limit.json())
|
||||
|
||||
# With max_tokens_per_doc
|
||||
response_with_limit = requests.post(
|
||||
server.url_for("rerank"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"query": query,
|
||||
"documents": [long_doc],
|
||||
"max_tokens_per_doc": 10,
|
||||
},
|
||||
)
|
||||
response_with_limit.raise_for_status()
|
||||
rerank_with_limit = RerankResponse.model_validate(response_with_limit.json())
|
||||
|
||||
assert rerank_with_limit.usage.prompt_tokens < rerank_no_limit.usage.prompt_tokens
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rerank_max_tokens_per_doc_validation(
|
||||
server: RemoteOpenAIServer,
|
||||
):
|
||||
"""Test that max_tokens_per_doc validation works correctly."""
|
||||
query = "What is the capital of France?"
|
||||
documents = ["The capital of France is Paris."]
|
||||
|
||||
# Test with max_tokens_per_doc=0 (should succeed — means no truncation)
|
||||
response = requests.post(
|
||||
server.url_for("rerank"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"query": query,
|
||||
"documents": documents,
|
||||
"max_tokens_per_doc": 0,
|
||||
},
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
# Test with invalid max_tokens_per_doc (negative)
|
||||
response = requests.post(
|
||||
server.url_for("rerank"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"query": query,
|
||||
"documents": documents,
|
||||
"max_tokens_per_doc": -5,
|
||||
},
|
||||
)
|
||||
assert response.status_code == 400
|
||||
assert "max_tokens_per_doc must be a non-negative integer" in response.text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("task", ["embed", "token_embed", "token_classify", "plugin"])
|
||||
async def test_pooling_not_supported(server: RemoteOpenAIServer, task: str):
|
||||
response = requests.post(
|
||||
server.url_for("pooling"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"input": input_text,
|
||||
"encoding_format": "float",
|
||||
"task": task,
|
||||
},
|
||||
)
|
||||
assert response.json()["error"]["type"] == "BadRequestError"
|
||||
if task == "plugin":
|
||||
err_msg = "No IOProcessor plugin installed."
|
||||
elif task == "token_classify":
|
||||
err_msg = "Try switching the model's pooling_task via"
|
||||
else:
|
||||
err_msg = f"Unsupported task: {task!r}"
|
||||
assert response.json()["error"]["message"].startswith(err_msg)
|
||||
@@ -1,488 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from tests.utils import VLLM_PATH, RemoteOpenAIServer
|
||||
from vllm.entrypoints.pooling.scoring.protocol import RerankResponse, ScoreResponse
|
||||
from vllm.multimodal.utils import encode_image_url, fetch_image
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
MODEL_NAME = "Qwen/Qwen3-VL-Reranker-2B"
|
||||
HF_OVERRIDES = {
|
||||
"architectures": ["Qwen3VLForSequenceClassification"],
|
||||
"classifier_from_token": ["no", "yes"],
|
||||
"is_original_qwen3_reranker": True,
|
||||
}
|
||||
|
||||
ROCM_ATTN_BACKENDS = [
|
||||
"ROCM_ATTN",
|
||||
"ROCM_AITER_FA",
|
||||
"TRITON_ATTN",
|
||||
"FLEX_ATTENTION",
|
||||
]
|
||||
|
||||
ATTN_BACKENDS = ROCM_ATTN_BACKENDS if current_platform.is_rocm() else ["auto"]
|
||||
|
||||
# The absolute floor matters only for the low ~0.10 text-vs-text score, where a
|
||||
# few thousandths of drift would otherwise exceed the relative bound.
|
||||
REL_TOL = 0.05
|
||||
ABS_TOL = 0.007
|
||||
|
||||
# ROCm: disable prefix caching and eliminate batch variance to reduce
|
||||
# test flakiness.
|
||||
ROCM_EXTRA_ARGS = (
|
||||
["--no-enable-prefix-caching", "--max-num-seqs", "1"]
|
||||
if current_platform.is_rocm()
|
||||
else []
|
||||
)
|
||||
|
||||
|
||||
def assert_score(actual: float, expected: float, backend: str, label: str):
|
||||
tol = REL_TOL
|
||||
abs_tol = ABS_TOL
|
||||
diff = abs(actual - expected)
|
||||
rel_diff = diff / abs(expected) if expected != 0 else diff
|
||||
print(
|
||||
f"[{backend}] {label}: actual={actual:.6f} expected={expected:.6f} "
|
||||
f"diff={diff:.6f} rel_diff={rel_diff:.4f} tol={tol} abs_tol={abs_tol}"
|
||||
)
|
||||
assert actual == pytest.approx(expected, rel=tol, abs=abs_tol), (
|
||||
f"[{backend}] {label}: score mismatch — "
|
||||
f"actual={actual:.6f}, expected={expected:.6f}, "
|
||||
f"rel_diff={rel_diff:.4f}, tol={tol}, abs_tol={abs_tol}"
|
||||
)
|
||||
|
||||
|
||||
query = "A cat standing in the snow."
|
||||
document = "This product was excellent and exceeded my expectations."
|
||||
image_url = "https://vllm-public-assets.s3.us-west-2.amazonaws.com/multimodal_asset/cat_snow.jpg"
|
||||
documents = [
|
||||
{
|
||||
"type": "text",
|
||||
"text": document,
|
||||
},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": image_url},
|
||||
},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": encode_image_url(fetch_image(image_url))},
|
||||
},
|
||||
]
|
||||
|
||||
# One bf16 ULP in attention compounds to ~0.1% here, so ROCm backends each
|
||||
# settle differently; these are their midpoint across backends and archs.
|
||||
if current_platform.is_rocm():
|
||||
TEXT_VS_TEXT = 0.105243
|
||||
TEXT_VS_IMAGE = 0.742472
|
||||
TEXT_VS_TEXT_PLUS_IMAGE = 0.544568
|
||||
else:
|
||||
TEXT_VS_TEXT = 0.10040374100208282
|
||||
TEXT_VS_IMAGE = 0.7423753142356873
|
||||
TEXT_VS_TEXT_PLUS_IMAGE = 0.5298863053321838
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", params=ATTN_BACKENDS)
|
||||
def server(request):
|
||||
backend = request.param
|
||||
print(f"\n=== Starting server with attention backend: {backend} ===")
|
||||
args = [
|
||||
"--enforce-eager",
|
||||
"--max-model-len",
|
||||
"8192",
|
||||
"--chat-template",
|
||||
str(VLLM_PATH / "examples/pooling/score/template/qwen3_vl_reranker.jinja"),
|
||||
]
|
||||
|
||||
env = dict()
|
||||
if backend != "auto":
|
||||
args += ["--attention-config", json.dumps({"backend": backend})]
|
||||
args += ROCM_EXTRA_ARGS
|
||||
|
||||
if backend != "ROCM_AITER_FA":
|
||||
env["VLLM_ROCM_USE_AITER"] = "0"
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
MODEL_NAME, args, override_hf_configs=HF_OVERRIDES, env_dict=env
|
||||
) as remote_server:
|
||||
print(f"=== Server ready with backend: {backend} ===")
|
||||
yield remote_server, backend
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_score_api_queries_str_documents_str(
|
||||
server: tuple[RemoteOpenAIServer, str],
|
||||
):
|
||||
remote_server, backend = server
|
||||
score_response = requests.post(
|
||||
remote_server.url_for("score"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"queries": query,
|
||||
"documents": document,
|
||||
},
|
||||
)
|
||||
score_response.raise_for_status()
|
||||
score = ScoreResponse.model_validate(score_response.json())
|
||||
|
||||
assert score.id is not None
|
||||
assert score.data is not None
|
||||
assert len(score.data) == 1
|
||||
assert score.usage.prompt_tokens == 81
|
||||
assert_score(score.data[0].score, TEXT_VS_TEXT, backend, "text_vs_text")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_score_api_queries_str_documents_text_content(
|
||||
server: tuple[RemoteOpenAIServer, str],
|
||||
):
|
||||
remote_server, backend = server
|
||||
score_response = requests.post(
|
||||
remote_server.url_for("score"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"queries": query,
|
||||
"documents": {"content": [documents[0]]},
|
||||
},
|
||||
)
|
||||
score_response.raise_for_status()
|
||||
score = ScoreResponse.model_validate(score_response.json())
|
||||
|
||||
assert score.id is not None
|
||||
assert score.data is not None
|
||||
assert len(score.data) == 1
|
||||
assert score.usage.prompt_tokens == 81
|
||||
assert_score(score.data[0].score, TEXT_VS_TEXT, backend, "text_vs_text")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_score_api_queries_str_documents_image_url_content(
|
||||
server: tuple[RemoteOpenAIServer, str],
|
||||
):
|
||||
remote_server, backend = server
|
||||
score_response = requests.post(
|
||||
remote_server.url_for("score"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"queries": query,
|
||||
"documents": {"content": [documents[1]]},
|
||||
},
|
||||
)
|
||||
score_response.raise_for_status()
|
||||
score = ScoreResponse.model_validate(score_response.json())
|
||||
|
||||
assert score.id is not None
|
||||
assert score.data is not None
|
||||
assert len(score.data) == 1
|
||||
assert score.usage.prompt_tokens == 98
|
||||
assert_score(score.data[0].score, TEXT_VS_IMAGE, backend, "text_vs_image")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_score_api_queries_str_documents_image_base64_content(
|
||||
server: tuple[RemoteOpenAIServer, str],
|
||||
):
|
||||
remote_server, backend = server
|
||||
score_response = requests.post(
|
||||
remote_server.url_for("score"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"queries": query,
|
||||
"documents": {"content": [documents[2]]},
|
||||
},
|
||||
)
|
||||
score_response.raise_for_status()
|
||||
score = ScoreResponse.model_validate(score_response.json())
|
||||
|
||||
assert score.id is not None
|
||||
assert score.data is not None
|
||||
assert len(score.data) == 1
|
||||
assert score.usage.prompt_tokens == 98
|
||||
assert_score(score.data[0].score, TEXT_VS_IMAGE, backend, "text_vs_image_base64")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_score_api_queries_str_documents_image_url_plus_text_content(
|
||||
server: tuple[RemoteOpenAIServer, str],
|
||||
):
|
||||
remote_server, backend = server
|
||||
score_response = requests.post(
|
||||
remote_server.url_for("score"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"queries": query,
|
||||
"documents": {"content": [documents[0], documents[1]]},
|
||||
},
|
||||
)
|
||||
score_response.raise_for_status()
|
||||
score = ScoreResponse.model_validate(score_response.json())
|
||||
|
||||
assert score.id is not None
|
||||
assert score.data is not None
|
||||
assert len(score.data) == 1
|
||||
assert score.usage.prompt_tokens == 107
|
||||
assert_score(
|
||||
score.data[0].score, TEXT_VS_TEXT_PLUS_IMAGE, backend, "text_vs_text_plus_image"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_score_api_queries_str_documents_list(
|
||||
server: tuple[RemoteOpenAIServer, str],
|
||||
):
|
||||
remote_server, backend = server
|
||||
score_response = requests.post(
|
||||
remote_server.url_for("score"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"queries": query,
|
||||
"documents": [
|
||||
document,
|
||||
{"content": [documents[0]]},
|
||||
{"content": [documents[1]]},
|
||||
{"content": [documents[0], documents[1]]},
|
||||
],
|
||||
},
|
||||
)
|
||||
score_response.raise_for_status()
|
||||
score = ScoreResponse.model_validate(score_response.json())
|
||||
|
||||
assert score.id is not None
|
||||
assert score.data is not None
|
||||
assert len(score.data) == 4
|
||||
assert score.usage.prompt_tokens == 367
|
||||
assert_score(score.data[0].score, TEXT_VS_TEXT, backend, "list[0]_text_vs_text")
|
||||
assert_score(score.data[1].score, TEXT_VS_TEXT, backend, "list[1]_text_vs_text")
|
||||
assert_score(score.data[2].score, TEXT_VS_IMAGE, backend, "list[2]_text_vs_image")
|
||||
assert_score(
|
||||
score.data[3].score,
|
||||
TEXT_VS_TEXT_PLUS_IMAGE,
|
||||
backend,
|
||||
"list[3]_text_vs_text_plus_image",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rerank_api_queries_str_documents_list(
|
||||
server: tuple[RemoteOpenAIServer, str],
|
||||
):
|
||||
remote_server, backend = server
|
||||
rerank_response = requests.post(
|
||||
remote_server.url_for("rerank"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"query": query,
|
||||
"documents": [
|
||||
document,
|
||||
{"content": [documents[0]]},
|
||||
{"content": [documents[1]]},
|
||||
{"content": [documents[0], documents[1]]},
|
||||
],
|
||||
},
|
||||
)
|
||||
rerank_response.raise_for_status()
|
||||
rerank = RerankResponse.model_validate(rerank_response.json())
|
||||
|
||||
assert rerank.id is not None
|
||||
assert rerank.model is not None
|
||||
assert rerank.usage is not None
|
||||
assert len(rerank.results) == 4
|
||||
|
||||
rerank.results.sort(key=lambda x: x.index)
|
||||
assert_score(
|
||||
rerank.results[0].relevance_score,
|
||||
TEXT_VS_TEXT,
|
||||
backend,
|
||||
"rerank[0]_text_vs_text",
|
||||
)
|
||||
assert_score(
|
||||
rerank.results[1].relevance_score,
|
||||
TEXT_VS_TEXT,
|
||||
backend,
|
||||
"rerank[1]_text_vs_text",
|
||||
)
|
||||
assert_score(
|
||||
rerank.results[2].relevance_score,
|
||||
TEXT_VS_IMAGE,
|
||||
backend,
|
||||
"rerank[2]_text_vs_image",
|
||||
)
|
||||
assert_score(
|
||||
rerank.results[3].relevance_score,
|
||||
TEXT_VS_TEXT_PLUS_IMAGE,
|
||||
backend,
|
||||
"rerank[3]_text_vs_text_plus_image",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_score_api_queries_list_documents_list(
|
||||
server: tuple[RemoteOpenAIServer, str],
|
||||
):
|
||||
remote_server, backend = server
|
||||
score_response = requests.post(
|
||||
remote_server.url_for("score"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"queries": [query] * 4,
|
||||
"documents": [
|
||||
document,
|
||||
{"content": [documents[0]]},
|
||||
{"content": [documents[1]]},
|
||||
{"content": [documents[0], documents[1]]},
|
||||
],
|
||||
},
|
||||
)
|
||||
score_response.raise_for_status()
|
||||
score = ScoreResponse.model_validate(score_response.json())
|
||||
|
||||
assert score.id is not None
|
||||
assert score.data is not None
|
||||
assert len(score.data) == 4
|
||||
assert score.usage.prompt_tokens == 367
|
||||
assert_score(score.data[0].score, TEXT_VS_TEXT, backend, "paired[0]_text_vs_text")
|
||||
assert_score(score.data[1].score, TEXT_VS_TEXT, backend, "paired[1]_text_vs_text")
|
||||
assert_score(score.data[2].score, TEXT_VS_IMAGE, backend, "paired[2]_text_vs_image")
|
||||
assert_score(
|
||||
score.data[3].score,
|
||||
TEXT_VS_TEXT_PLUS_IMAGE,
|
||||
backend,
|
||||
"paired[3]_text_vs_text_plus_image",
|
||||
)
|
||||
|
||||
|
||||
INSTRUCTION = (
|
||||
"Given a multimodal retrieval query, retrieve candidates that "
|
||||
"visually or textually match the requested scene, object, or action."
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_score_api_instruction_field(
|
||||
server: tuple[RemoteOpenAIServer, str],
|
||||
):
|
||||
remote_server, _ = server
|
||||
|
||||
default_response = requests.post(
|
||||
remote_server.url_for("score"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"queries": query,
|
||||
"documents": document,
|
||||
},
|
||||
)
|
||||
default_response.raise_for_status()
|
||||
default_score = ScoreResponse.model_validate(default_response.json())
|
||||
|
||||
instruction_response = requests.post(
|
||||
remote_server.url_for("score"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"queries": query,
|
||||
"documents": document,
|
||||
"instruction": INSTRUCTION,
|
||||
},
|
||||
)
|
||||
instruction_response.raise_for_status()
|
||||
instruction_score = ScoreResponse.model_validate(instruction_response.json())
|
||||
|
||||
assert instruction_score.id is not None
|
||||
assert instruction_score.data is not None
|
||||
assert len(instruction_score.data) == 1
|
||||
assert instruction_score.usage.prompt_tokens > default_score.usage.prompt_tokens
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rerank_api_instruction_field(
|
||||
server: tuple[RemoteOpenAIServer, str],
|
||||
):
|
||||
remote_server, _ = server
|
||||
|
||||
doc_list = [
|
||||
document,
|
||||
{"content": [documents[0]]},
|
||||
{"content": [documents[1]]},
|
||||
{"content": [documents[0], documents[1]]},
|
||||
]
|
||||
|
||||
default_response = requests.post(
|
||||
remote_server.url_for("rerank"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"query": query,
|
||||
"documents": doc_list,
|
||||
},
|
||||
)
|
||||
default_response.raise_for_status()
|
||||
default_rerank = RerankResponse.model_validate(default_response.json())
|
||||
|
||||
instruction_response = requests.post(
|
||||
remote_server.url_for("rerank"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"query": query,
|
||||
"documents": doc_list,
|
||||
"instruction": INSTRUCTION,
|
||||
},
|
||||
)
|
||||
instruction_response.raise_for_status()
|
||||
instruction_rerank = RerankResponse.model_validate(instruction_response.json())
|
||||
|
||||
assert instruction_rerank.id is not None
|
||||
assert instruction_rerank.model is not None
|
||||
assert instruction_rerank.usage is not None
|
||||
assert len(instruction_rerank.results) == len(default_rerank.results)
|
||||
assert instruction_rerank.usage.prompt_tokens > default_rerank.usage.prompt_tokens
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rerank_api_instruction_field_matches_chat_template_kwargs(
|
||||
server: tuple[RemoteOpenAIServer, str],
|
||||
):
|
||||
remote_server, backend = server
|
||||
|
||||
doc_list = [
|
||||
document,
|
||||
{"content": [documents[0]]},
|
||||
{"content": [documents[1]]},
|
||||
{"content": [documents[0], documents[1]]},
|
||||
]
|
||||
|
||||
field_response = requests.post(
|
||||
remote_server.url_for("rerank"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"query": query,
|
||||
"documents": doc_list,
|
||||
"instruction": INSTRUCTION,
|
||||
},
|
||||
)
|
||||
field_response.raise_for_status()
|
||||
field_rerank = RerankResponse.model_validate(field_response.json())
|
||||
|
||||
kwargs_response = requests.post(
|
||||
remote_server.url_for("rerank"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"query": query,
|
||||
"documents": doc_list,
|
||||
"chat_template_kwargs": {"instruction": INSTRUCTION},
|
||||
},
|
||||
)
|
||||
kwargs_response.raise_for_status()
|
||||
kwargs_rerank = RerankResponse.model_validate(kwargs_response.json())
|
||||
|
||||
assert kwargs_rerank.usage.prompt_tokens == field_rerank.usage.prompt_tokens
|
||||
|
||||
field_scores = [
|
||||
r.relevance_score for r in sorted(field_rerank.results, key=lambda x: x.index)
|
||||
]
|
||||
kwargs_scores = [
|
||||
r.relevance_score for r in sorted(kwargs_rerank.results, key=lambda x: x.index)
|
||||
]
|
||||
assert field_scores == pytest.approx(kwargs_scores, rel=REL_TOL, abs=ABS_TOL)
|
||||
@@ -1,247 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for ScoringIOProcessor helpers."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from tokenizers import Tokenizer, models, pre_tokenizers, processors
|
||||
from transformers import TokenizersBackend
|
||||
|
||||
from vllm import PoolingParams, TokensPrompt
|
||||
from vllm.entrypoints.pooling.scoring.io_processor import (
|
||||
BiEncoderIOProcessor,
|
||||
CrossEncoderIOProcessor,
|
||||
JinaRankingIOProcessor,
|
||||
LateInteractionIOProcessor,
|
||||
_apply_post_tokenization_to_token_type_ids,
|
||||
)
|
||||
from vllm.entrypoints.pooling.scoring.typing import ScoringData
|
||||
from vllm.entrypoints.pooling.scoring.utils import compress_token_type_ids
|
||||
from vllm.entrypoints.pooling.typing import OfflineScoringInputsContext
|
||||
from vllm.outputs import PoolingOutput, PoolingRequestOutput, RequestError
|
||||
from vllm.renderers import TokenizeParams
|
||||
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
|
||||
|
||||
@dataclass
|
||||
class _DummyTokenizer:
|
||||
truncation_side: str = "left"
|
||||
# Outside the range of the prompt ids below, so a test can tell a pad
|
||||
# token apart from a real one.
|
||||
pad_token_id: int = 99999
|
||||
pad_token_type_id: int = 0
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def llm_reranker_processor() -> CrossEncoderIOProcessor:
|
||||
vocab = {
|
||||
"[UNK]": 0,
|
||||
"[CLS]": 1,
|
||||
"[SEP]": 2,
|
||||
"a": 3,
|
||||
"bc": 4,
|
||||
"##b": 5,
|
||||
"##c": 6,
|
||||
}
|
||||
backend = Tokenizer(models.WordPiece(vocab, unk_token="[UNK]"))
|
||||
backend.pre_tokenizer = pre_tokenizers.WhitespaceSplit()
|
||||
backend.post_processor = processors.TemplateProcessing(
|
||||
single="[CLS] $A [SEP]",
|
||||
special_tokens=[("[CLS]", 1), ("[SEP]", 2)],
|
||||
)
|
||||
|
||||
processor = CrossEncoderIOProcessor.__new__(CrossEncoderIOProcessor)
|
||||
processor.model_config = SimpleNamespace(enable_prompt_embeds=False)
|
||||
processor.tokenizer = TokenizersBackend(
|
||||
tokenizer_object=backend,
|
||||
unk_token="[UNK]",
|
||||
cls_token="[CLS]",
|
||||
sep_token="[SEP]",
|
||||
)
|
||||
processor.supports_score_template = False
|
||||
processor.use_sep_token = False
|
||||
processor.model = None
|
||||
return processor
|
||||
|
||||
|
||||
def test_llm_reranker_tokenization_is_independent_of_nonbinding_doc_limit(
|
||||
llm_reranker_processor: CrossEncoderIOProcessor,
|
||||
):
|
||||
encode_kwargs = {"add_special_tokens": True}
|
||||
|
||||
_, uncapped = llm_reranker_processor.get_score_prompt("a", "bc", encode_kwargs)
|
||||
_, nonbinding = llm_reranker_processor.get_score_prompt(
|
||||
"a", "bc", encode_kwargs, max_tokens_per_doc=10
|
||||
)
|
||||
|
||||
expected = [1, 3, 5, 6, 2]
|
||||
assert uncapped["prompt_token_ids"] == expected
|
||||
assert nonbinding["prompt_token_ids"] == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"default_template,request_template,expected",
|
||||
[
|
||||
("{{ messages[0].content }} {{ messages[1].content }}", None, "a bc"),
|
||||
(
|
||||
"{{ messages[0].content }} {{ messages[1].content }}",
|
||||
"{{ messages[1].content }} {{ messages[0].content }}",
|
||||
"bc a",
|
||||
),
|
||||
(None, "{{ messages[1].content }} {{ messages[0].content }}", "bc a"),
|
||||
(None, None, "abc"),
|
||||
],
|
||||
)
|
||||
def test_offline_scoring_uses_default_template_unless_overridden(
|
||||
llm_reranker_processor, default_template, request_template, expected
|
||||
):
|
||||
processor = llm_reranker_processor
|
||||
processor.chat_template = default_template
|
||||
processor.renderer = SimpleNamespace(
|
||||
default_cmpl_tok_params=TokenizeParams(max_total_tokens=32)
|
||||
)
|
||||
ctx = OfflineScoringInputsContext(
|
||||
pooling_task="classify",
|
||||
tokenization_kwargs=None,
|
||||
lora_request=None,
|
||||
priorities=None,
|
||||
scoring_data=ScoringData(data_1=["a"], data_2=["bc"]),
|
||||
chat_template=request_template,
|
||||
pooling_params=PoolingParams(),
|
||||
)
|
||||
factory, _ = processor.get_request_factory_offline(ctx)
|
||||
(request,) = factory()
|
||||
text, prompt = processor.get_score_prompt(
|
||||
request["data_1"],
|
||||
request["data_2"],
|
||||
request["tok_params"].get_encode_kwargs(),
|
||||
chat_template=request["chat_template"],
|
||||
)
|
||||
|
||||
assert text == expected
|
||||
assert prompt["prompt_token_ids"] == processor.tokenizer.encode(expected)
|
||||
|
||||
|
||||
def test_token_type_ids_stay_aligned_with_a_truncated_padded_prompt():
|
||||
"""The cross-encoder segment boundary must survive truncate + pad.
|
||||
|
||||
`token_type_ids` are parallel to `prompt_token_ids` and are reduced to the
|
||||
document segment's start and end by `compress_token_type_ids`. If the two
|
||||
arrays are truncated and padded in different orders they no longer
|
||||
describe the same positions, and the model is told the query segment is
|
||||
empty.
|
||||
"""
|
||||
tokenizer = _DummyTokenizer()
|
||||
num_query, num_doc = 20, 30
|
||||
prompt = TokensPrompt(prompt_token_ids=list(range(num_query + num_doc)))
|
||||
token_type_ids = [0] * num_query + [1] * num_doc
|
||||
|
||||
tok_params = TokenizeParams(
|
||||
max_total_tokens=100,
|
||||
pad_prompt_tokens=-1,
|
||||
truncate_prompt_tokens=40,
|
||||
truncation_side="left",
|
||||
)
|
||||
|
||||
processed_prompt = tok_params.apply_post_tokenization(
|
||||
tokenizer,
|
||||
prompt,
|
||||
)
|
||||
assert "prompt_token_ids" in processed_prompt
|
||||
token_type_ids = _apply_post_tokenization_to_token_type_ids(
|
||||
tokenizer, tok_params, token_type_ids
|
||||
)
|
||||
|
||||
prompt_token_ids = processed_prompt["prompt_token_ids"]
|
||||
assert len(token_type_ids) == len(prompt_token_ids)
|
||||
|
||||
# Keeping the last 40 tokens drops the first 10 query tokens, so 10 query
|
||||
# tokens survive and the document starts at index 10.
|
||||
doc_start, doc_end = compress_token_type_ids(token_type_ids)
|
||||
assert (doc_start, doc_end) == (10, 40)
|
||||
assert prompt_token_ids[:doc_start] == list(range(10, num_query))
|
||||
assert prompt_token_ids[doc_start:doc_end] == list(range(num_query, 50))
|
||||
assert token_type_ids[doc_end:] == [tokenizer.pad_token_type_id] * 60
|
||||
|
||||
|
||||
def test_token_type_padding_uses_the_tokenizer_pad_type():
|
||||
tokenizer = _DummyTokenizer()
|
||||
tok_params = TokenizeParams(
|
||||
max_total_tokens=8,
|
||||
pad_prompt_tokens=8,
|
||||
truncate_prompt_tokens=None,
|
||||
truncation_side="right",
|
||||
)
|
||||
|
||||
token_type_ids = _apply_post_tokenization_to_token_type_ids(
|
||||
tokenizer, tok_params, [0, 0, 0, 1, 1]
|
||||
)
|
||||
|
||||
assert token_type_ids == [0, 0, 0, 1, 1, 0, 0, 0]
|
||||
assert compress_token_type_ids(token_type_ids) == (3, 5)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("token_type_ids", [[0, 1, 0, 1], [0, 2]])
|
||||
def test_compress_token_type_ids_rejects_invalid_sequences(token_type_ids):
|
||||
with pytest.raises(ValueError, match="optional trailing zeros"):
|
||||
compress_token_type_ids(token_type_ids)
|
||||
|
||||
|
||||
def _pooling_output(
|
||||
request_id: str,
|
||||
data: torch.Tensor,
|
||||
error: RequestError | None = None,
|
||||
) -> PoolingRequestOutput:
|
||||
return PoolingRequestOutput(
|
||||
request_id=request_id,
|
||||
outputs=PoolingOutput(data),
|
||||
prompt_token_ids=[1],
|
||||
num_cached_tokens=0,
|
||||
finished=True,
|
||||
error=error,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"processor_type", [BiEncoderIOProcessor, LateInteractionIOProcessor]
|
||||
)
|
||||
@pytest.mark.parametrize("failed_index", [0, 1])
|
||||
def test_paired_scoring_preserves_request_error(processor_type, failed_index):
|
||||
processor = processor_type.__new__(processor_type)
|
||||
processor.pad_token_id = None
|
||||
error = RequestError(
|
||||
code="multimodal_cache_miss",
|
||||
message="Multi-modal processor cache miss.",
|
||||
retryable=True,
|
||||
)
|
||||
outputs = [
|
||||
_pooling_output("query", torch.tensor([1.0, 2.0])),
|
||||
_pooling_output("document", torch.tensor([3.0, 4.0])),
|
||||
]
|
||||
outputs[failed_index] = _pooling_output(
|
||||
outputs[failed_index].request_id,
|
||||
torch.empty(0),
|
||||
error,
|
||||
)
|
||||
|
||||
(result,) = processor._post_process(outputs, n_queries=1)
|
||||
|
||||
assert result.request_id == "query_document"
|
||||
assert result.error is error
|
||||
assert result.outputs is outputs[failed_index].outputs
|
||||
|
||||
|
||||
def test_jina_scoring_preserves_request_error_before_indexing_output():
|
||||
processor = JinaRankingIOProcessor.__new__(JinaRankingIOProcessor)
|
||||
error = RequestError(
|
||||
code="multimodal_cache_miss",
|
||||
message="Multi-modal processor cache miss.",
|
||||
retryable=True,
|
||||
)
|
||||
failed = _pooling_output("request", torch.empty(0), error)
|
||||
|
||||
assert processor._post_process([failed], n_queries=1) == [failed]
|
||||
@@ -1,150 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for JinaRankingIOProcessor request building."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from tokenizers import Tokenizer, models, pre_tokenizers
|
||||
from transformers import TokenizersBackend
|
||||
|
||||
from vllm import PoolingParams
|
||||
from vllm.entrypoints.pooling.base.io_processor import PoolingIOProcessor
|
||||
from vllm.entrypoints.pooling.scoring.io_processor import JinaRankingIOProcessor
|
||||
from vllm.entrypoints.pooling.scoring.protocol import RerankRequest
|
||||
from vllm.entrypoints.pooling.scoring.typing import ScoringData
|
||||
from vllm.entrypoints.pooling.typing import OfflineScoringInputsContext
|
||||
from vllm.renderers import TokenizeParams
|
||||
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
|
||||
|
||||
def test_online_forwards_truncate_prompt_tokens_to_proxy(monkeypatch):
|
||||
"""The proxy request handed to the base factory must carry
|
||||
truncate_prompt_tokens/truncation_side from the real request.
|
||||
|
||||
JinaRankingIOProcessor swaps ctx.request for a proxy
|
||||
PoolingCompletionRequest before delegating to the base factory, which
|
||||
reads truncation off ctx.request. Dropping the fields on the proxy
|
||||
silently disables truncate_prompt_tokens for Jina rerank/score.
|
||||
"""
|
||||
proc = JinaRankingIOProcessor.__new__(JinaRankingIOProcessor)
|
||||
monkeypatch.setattr(
|
||||
proc,
|
||||
"valid_inputs_online",
|
||||
MagicMock(return_value=ScoringData(data_1=["query"], data_2=["doc"])),
|
||||
)
|
||||
monkeypatch.setattr(proc, "_get_token_limits", MagicMock(return_value=(0, 0)))
|
||||
monkeypatch.setattr(
|
||||
proc, "ensure_str", MagicMock(side_effect=lambda data: list(data))
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
proc, "format_docs_prompts_func", MagicMock(return_value="formatted prompt")
|
||||
)
|
||||
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
def _spy_base(self, ctx):
|
||||
captured["truncate_prompt_tokens"] = ctx.request.truncate_prompt_tokens
|
||||
captured["truncation_side"] = ctx.request.truncation_side
|
||||
return []
|
||||
|
||||
monkeypatch.setattr(PoolingIOProcessor, "get_request_factory_online", _spy_base)
|
||||
|
||||
request = RerankRequest(
|
||||
model="m",
|
||||
query="query",
|
||||
documents=["doc"],
|
||||
truncate_prompt_tokens=512,
|
||||
truncation_side="left",
|
||||
)
|
||||
ctx = MagicMock()
|
||||
ctx.request = request
|
||||
ctx.prompt_extras = None
|
||||
|
||||
proc.get_request_factory_online(ctx)
|
||||
|
||||
assert captured["truncate_prompt_tokens"] == 512
|
||||
assert captured["truncation_side"] == "left"
|
||||
# The real request is restored after delegating.
|
||||
assert ctx.request is request
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def offline_processor_and_context():
|
||||
backend = Tokenizer(models.WordLevel({"[UNK]": 0}, unk_token="[UNK]"))
|
||||
backend.pre_tokenizer = pre_tokenizers.WhitespaceSplit()
|
||||
proc = JinaRankingIOProcessor.__new__(JinaRankingIOProcessor)
|
||||
proc.tokenizer = TokenizersBackend(tokenizer_object=backend)
|
||||
proc.model_config = SimpleNamespace(max_model_len=1024, is_encoder_decoder=False)
|
||||
proc.renderer = SimpleNamespace(
|
||||
default_cmpl_tok_params=TokenizeParams(max_total_tokens=1024)
|
||||
)
|
||||
ctx = OfflineScoringInputsContext(
|
||||
pooling_task="token_embed",
|
||||
scoring_data=ScoringData(
|
||||
data_1=["query alpha beta", "query gamma delta"],
|
||||
data_2=["document one two", "document three four"],
|
||||
),
|
||||
pooling_params=PoolingParams(
|
||||
extra_kwargs={
|
||||
"chat_template_kwargs": {"instruction": "Keep this instruction"}
|
||||
}
|
||||
),
|
||||
tokenization_kwargs=None,
|
||||
chat_template=None,
|
||||
lora_request=None,
|
||||
priorities=None,
|
||||
)
|
||||
return proc, ctx
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("query_limit", "doc_limit", "n_queries"),
|
||||
[(None, None, 1), (0, 0, 2), (1, None, 1), (None, 2, 1), (1, 2, 1), (1, 2, 2)],
|
||||
ids=["unset", "zero", "query", "doc", "both-1-to-n", "both-n-to-n"],
|
||||
)
|
||||
def test_offline_truncates_text_before_formatting(
|
||||
offline_processor_and_context, query_limit, doc_limit, n_queries
|
||||
):
|
||||
proc, ctx = offline_processor_and_context
|
||||
queries = ctx.scoring_data.data_1[:n_queries]
|
||||
docs = ctx.scoring_data.data_2
|
||||
ctx.scoring_data = ScoringData(data_1=queries, data_2=docs)
|
||||
ctx.pooling_params.extra_kwargs.update(
|
||||
{
|
||||
name: limit
|
||||
for name, limit in (
|
||||
("max_tokens_per_query", query_limit),
|
||||
("max_tokens_per_doc", doc_limit),
|
||||
)
|
||||
if limit is not None
|
||||
}
|
||||
)
|
||||
|
||||
factory, num_requests = proc.get_request_factory_offline(ctx)
|
||||
requests = list(factory())
|
||||
assert len(requests) == num_requests == n_queries
|
||||
for i, request in enumerate(requests):
|
||||
prompt = request["prompts"]["prompt"]
|
||||
query = " ".join(queries[i].split()[: query_limit or None])
|
||||
assert f"<query>\n{query}<|rerank_token|>\n</query>" in prompt
|
||||
prompt_docs = docs if n_queries == 1 else [docs[i]]
|
||||
for j, doc in enumerate(prompt_docs):
|
||||
doc = " ".join(doc.split()[: doc_limit or None])
|
||||
assert f'<passage id="{j}">\n{doc}<|embed_token|>\n</passage>' in prompt
|
||||
assert prompt.count("<|embed_token|>") == len(prompt_docs)
|
||||
assert prompt.count("<|rerank_token|>") == 1
|
||||
assert "<instruct>\nKeep this instruction\n</instruct>" in prompt
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", ["max_tokens_per_query", "max_tokens_per_doc"])
|
||||
@pytest.mark.parametrize("limit", [-1, 1024])
|
||||
def test_offline_rejects_invalid_token_limits(
|
||||
offline_processor_and_context, name, limit
|
||||
):
|
||||
proc, ctx = offline_processor_and_context
|
||||
ctx.pooling_params.extra_kwargs[name] = limit
|
||||
with pytest.raises(ValueError, match=name):
|
||||
proc.get_request_factory_offline(ctx)
|
||||
@@ -1,105 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import weakref
|
||||
|
||||
import pytest
|
||||
|
||||
from .util import ColBERTScoringHfRunner
|
||||
|
||||
MODEL_NAME = "answerdotai/answerai-colbert-small-v1"
|
||||
COLBERT_DIM = 96
|
||||
|
||||
LINEAR_WEIGHTS_KEY = "linear.weight"
|
||||
PROMPT = "The chef prepared a delicious meal."
|
||||
|
||||
TEXTS_1 = [
|
||||
"What is the capital of France?",
|
||||
"What is the capital of Germany?",
|
||||
]
|
||||
|
||||
TEXTS_2 = [
|
||||
"The capital of France is Paris.",
|
||||
"The capital of Germany is Berlin.",
|
||||
]
|
||||
|
||||
DTYPE = "half"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def llm(vllm_runner):
|
||||
with vllm_runner(
|
||||
MODEL_NAME,
|
||||
max_model_len=None,
|
||||
max_num_batched_tokens=32768,
|
||||
tensor_parallel_size=1,
|
||||
gpu_memory_utilization=0.75,
|
||||
enforce_eager=True,
|
||||
seed=0,
|
||||
enable_chunked_prefill=None,
|
||||
) as runner:
|
||||
# pytest caches yielded fixtures until after teardown, so use a proxy to
|
||||
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
|
||||
yield weakref.proxy(runner.llm)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def hf_model():
|
||||
return ColBERTScoringHfRunner(
|
||||
model_name=MODEL_NAME, linear_weights_key=LINEAR_WEIGHTS_KEY
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_1_to_1(llm, hf_model):
|
||||
text_pair = [TEXTS_1[0], TEXTS_2[0]]
|
||||
|
||||
hf_outputs = hf_model.predict([text_pair]).tolist()
|
||||
vllm_outputs = [
|
||||
output.outputs.score for output in llm.score(text_pair[0], text_pair[1])
|
||||
]
|
||||
|
||||
assert len(vllm_outputs) == 1
|
||||
assert len(hf_outputs) == 1
|
||||
|
||||
assert hf_outputs[0] == pytest.approx(vllm_outputs[0], rel=0.01)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_1_to_n(llm, hf_model):
|
||||
text_pairs = [
|
||||
[TEXTS_1[0], TEXTS_2[0]],
|
||||
[TEXTS_1[0], TEXTS_2[1]],
|
||||
]
|
||||
|
||||
hf_outputs = hf_model.predict(text_pairs).tolist()
|
||||
vllm_outputs = [output.outputs.score for output in llm.score(TEXTS_1[0], TEXTS_2)]
|
||||
|
||||
assert len(vllm_outputs) == 2
|
||||
assert len(hf_outputs) == 2
|
||||
|
||||
assert hf_outputs[0] == pytest.approx(vllm_outputs[0], rel=0.01)
|
||||
assert hf_outputs[1] == pytest.approx(vllm_outputs[1], rel=0.01)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_n_to_n(llm, hf_model):
|
||||
text_pairs = [
|
||||
[TEXTS_1[0], TEXTS_2[0]],
|
||||
[TEXTS_1[1], TEXTS_2[1]],
|
||||
]
|
||||
|
||||
hf_outputs = hf_model.predict(text_pairs).tolist()
|
||||
vllm_outputs = [output.outputs.score for output in llm.score(TEXTS_1, TEXTS_2)]
|
||||
|
||||
assert len(vllm_outputs) == 2
|
||||
assert len(hf_outputs) == 2
|
||||
|
||||
assert hf_outputs[0] == pytest.approx(vllm_outputs[0], rel=0.01)
|
||||
assert hf_outputs[1] == pytest.approx(vllm_outputs[1], rel=0.01)
|
||||
|
||||
|
||||
def test_token_embed(llm):
|
||||
outputs = llm.encode(PROMPT, pooling_task="token_embed", use_tqdm=False)
|
||||
assert len(outputs) == 1
|
||||
assert outputs[0].outputs.data.shape == (9, COLBERT_DIM)
|
||||
@@ -1,79 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import weakref
|
||||
|
||||
import pytest
|
||||
|
||||
from .util import make_base64_image, make_image_mm_param
|
||||
|
||||
MODEL_NAME = "vidore/colpali-v1.3-hf"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def llm(vllm_runner):
|
||||
with vllm_runner(
|
||||
MODEL_NAME,
|
||||
max_model_len=None,
|
||||
max_num_batched_tokens=32768,
|
||||
tensor_parallel_size=1,
|
||||
gpu_memory_utilization=0.75,
|
||||
enforce_eager=True,
|
||||
seed=0,
|
||||
enable_chunked_prefill=None,
|
||||
) as runner:
|
||||
# pytest caches yielded fixtures until after teardown, so use a proxy to
|
||||
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
|
||||
yield weakref.proxy(runner.llm)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_query_text_vs_docs_image(llm):
|
||||
"""Score a text query against image documents via the multimodal path."""
|
||||
red_image = make_base64_image(64, 64, color=(255, 0, 0))
|
||||
blue_image = make_base64_image(64, 64, color=(0, 0, 255))
|
||||
|
||||
query = "Describe the red object"
|
||||
image_docs = [
|
||||
make_image_mm_param(red_image),
|
||||
make_image_mm_param(blue_image),
|
||||
]
|
||||
|
||||
scores = llm.score(query, image_docs)
|
||||
|
||||
assert len(scores) == 2
|
||||
assert scores[0].outputs.score > scores[1].outputs.score
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_query_text_vs_docs_mix(llm) -> None:
|
||||
"""Score a text query against a mix of text and image documents."""
|
||||
red_image = make_base64_image(64, 64, color=(255, 0, 0))
|
||||
|
||||
query = "What is the capital of France?"
|
||||
documents: list = [
|
||||
"The capital of France is Paris.",
|
||||
make_image_mm_param(red_image),
|
||||
]
|
||||
|
||||
scores = llm.score(query, documents)
|
||||
|
||||
assert len(scores) == 2
|
||||
assert scores[0].outputs.score > scores[1].outputs.score
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_query_image_vs_docs_text(llm) -> None:
|
||||
"""Score an image query against text documents."""
|
||||
red_image = make_base64_image(64, 64, color=(255, 0, 0))
|
||||
image_query = make_image_mm_param(red_image, text="red color")
|
||||
|
||||
documents = [
|
||||
"Describe the red object.",
|
||||
"The capital of France is Paris.",
|
||||
]
|
||||
|
||||
scores = llm.score(image_query, documents)
|
||||
|
||||
assert len(scores) == 2
|
||||
assert scores[0].outputs.score > scores[1].outputs.score
|
||||
@@ -1,267 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Online API tests for ColBERT late interaction scoring."""
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.entrypoints.pooling.scoring.protocol import RerankResponse, ScoreResponse
|
||||
|
||||
from .util import ColBERTScoringHfRunner
|
||||
|
||||
MODEL_NAME = "answerdotai/answerai-colbert-small-v1"
|
||||
COLBERT_DIM = 96
|
||||
MAX_MODEL_LEN = 512
|
||||
LINEAR_WEIGHTS_KEY = "linear.weight"
|
||||
|
||||
TEXTS_1 = [
|
||||
"What is the capital of France?",
|
||||
"What is the capital of Germany?",
|
||||
]
|
||||
|
||||
TEXTS_2 = [
|
||||
"The capital of France is Paris.",
|
||||
"The capital of Germany is Berlin.",
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_flash_late_interaction_rejects_mm_processor_kwargs_by_default():
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from vllm.config.multimodal import MultiModalConfig
|
||||
from vllm.entrypoints.pooling.base.serving import PoolingBaseServing
|
||||
from vllm.entrypoints.pooling.scoring.protocol import ScoreQueriesDocumentsRequest
|
||||
from vllm.entrypoints.pooling.scoring.serving import ServingScores
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
|
||||
serving = MagicMock()
|
||||
serving.model_config = MagicMock(multimodal_config=MultiModalConfig())
|
||||
serving.trust_request_mm_kwargs = False
|
||||
serving.io_processor = MagicMock()
|
||||
serving._init_ctx = PoolingBaseServing._init_ctx.__get__(serving)
|
||||
request = ScoreQueriesDocumentsRequest(
|
||||
model=MODEL_NAME,
|
||||
queries=TEXTS_1[0],
|
||||
documents=TEXTS_2[0],
|
||||
mm_processor_kwargs={"vision_min_num_patches": 3_000_000_000},
|
||||
)
|
||||
|
||||
with pytest.raises(
|
||||
VLLMValidationError,
|
||||
match="Per-request mm_processor_kwargs are disabled",
|
||||
):
|
||||
await ServingScores.flash_late_interaction(serving, request)
|
||||
serving._preprocessing.assert_not_called()
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", params=[True, False])
|
||||
def server(request):
|
||||
args = [
|
||||
"--max-model-len",
|
||||
str(MAX_MODEL_LEN),
|
||||
]
|
||||
|
||||
# Test run pooling score MaxSim on worker side (GPU)
|
||||
# aka flash-late-interaction
|
||||
if not request.param:
|
||||
args += ["--no-enable-flash-late-interaction"]
|
||||
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def hf_model():
|
||||
return ColBERTScoringHfRunner(
|
||||
model_name=MODEL_NAME, linear_weights_key=LINEAR_WEIGHTS_KEY
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_score_api_queries_str_1_documents_str_1(
|
||||
hf_model, server: RemoteOpenAIServer
|
||||
):
|
||||
score_response = requests.post(
|
||||
server.url_for("score"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"queries": TEXTS_1[0],
|
||||
"documents": TEXTS_2[0],
|
||||
},
|
||||
)
|
||||
score_response.raise_for_status()
|
||||
score = ScoreResponse.model_validate(score_response.json())
|
||||
|
||||
assert score.id is not None
|
||||
assert score.data is not None
|
||||
assert len(score.data) == 1
|
||||
|
||||
vllm_outputs = [d.score for d in score.data]
|
||||
hf_outputs = hf_model.predict([[TEXTS_1[0], TEXTS_2[0]]]).tolist()
|
||||
|
||||
for i in range(len(vllm_outputs)):
|
||||
assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_score_api_queries_str_1_documents_str_n(
|
||||
hf_model, server: RemoteOpenAIServer
|
||||
):
|
||||
text_pairs = [
|
||||
[TEXTS_1[0], TEXTS_2[0]],
|
||||
[TEXTS_1[0], TEXTS_2[1]],
|
||||
]
|
||||
|
||||
score_response = requests.post(
|
||||
server.url_for("score"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"queries": TEXTS_1[0],
|
||||
"documents": TEXTS_2,
|
||||
},
|
||||
)
|
||||
score_response.raise_for_status()
|
||||
score = ScoreResponse.model_validate(score_response.json())
|
||||
|
||||
assert score.id is not None
|
||||
assert score.data is not None
|
||||
assert len(score.data) == 2
|
||||
|
||||
vllm_outputs = [d.score for d in score.data]
|
||||
hf_outputs = hf_model.predict(text_pairs).tolist()
|
||||
|
||||
for i in range(len(vllm_outputs)):
|
||||
assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_score_api_queries_str_n_documents_str_n(
|
||||
hf_model, server: RemoteOpenAIServer
|
||||
):
|
||||
text_pairs = [
|
||||
[TEXTS_1[0], TEXTS_2[0]],
|
||||
[TEXTS_1[1], TEXTS_2[1]],
|
||||
]
|
||||
|
||||
score_response = requests.post(
|
||||
server.url_for("score"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"queries": TEXTS_1,
|
||||
"documents": TEXTS_2,
|
||||
},
|
||||
)
|
||||
score_response.raise_for_status()
|
||||
score = ScoreResponse.model_validate(score_response.json())
|
||||
|
||||
assert score.id is not None
|
||||
assert score.data is not None
|
||||
assert len(score.data) == 2
|
||||
|
||||
vllm_outputs = [d.score for d in score.data]
|
||||
hf_outputs = hf_model.predict(text_pairs).tolist()
|
||||
|
||||
for i in range(len(vllm_outputs)):
|
||||
assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rerank_api_texts(server: RemoteOpenAIServer):
|
||||
"""Test ColBERT rerank endpoint."""
|
||||
query = "What is the capital of France?"
|
||||
documents = [
|
||||
"The capital of Brazil is Brasilia.",
|
||||
"The capital of France is Paris.",
|
||||
]
|
||||
|
||||
rerank_response = requests.post(
|
||||
server.url_for("rerank"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"query": query,
|
||||
"documents": documents,
|
||||
},
|
||||
)
|
||||
rerank_response.raise_for_status()
|
||||
rerank = RerankResponse.model_validate(rerank_response.json())
|
||||
|
||||
assert rerank.id is not None
|
||||
assert rerank.results is not None
|
||||
assert len(rerank.results) == 2
|
||||
|
||||
paris_result = next(r for r in rerank.results if r.index == 1)
|
||||
brazil_result = next(r for r in rerank.results if r.index == 0)
|
||||
|
||||
assert paris_result.relevance_score > brazil_result.relevance_score
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rerank_api_top_n(server: RemoteOpenAIServer):
|
||||
"""Test ColBERT rerank with top_n parameter."""
|
||||
query = "What is the capital of France?"
|
||||
documents = [
|
||||
"The capital of Brazil is Brasilia.",
|
||||
"The capital of France is Paris.",
|
||||
"Machine learning is a field of AI.",
|
||||
]
|
||||
|
||||
rerank_response = requests.post(
|
||||
server.url_for("rerank"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"query": query,
|
||||
"documents": documents,
|
||||
"top_n": 2,
|
||||
},
|
||||
)
|
||||
rerank_response.raise_for_status()
|
||||
rerank = RerankResponse.model_validate(rerank_response.json())
|
||||
|
||||
assert len(rerank.results) == 2
|
||||
assert rerank.results[0].index == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_token_embed(server: RemoteOpenAIServer):
|
||||
"""Test ColBERT token_embed task via pooling endpoint."""
|
||||
text = "What is the capital of France?"
|
||||
|
||||
pooling_response = requests.post(
|
||||
server.url_for("pooling"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"input": text,
|
||||
"task": "token_embed",
|
||||
},
|
||||
)
|
||||
pooling_response.raise_for_status()
|
||||
pooling = pooling_response.json()
|
||||
|
||||
assert "data" in pooling
|
||||
assert len(pooling["data"]) == 1
|
||||
|
||||
embeddings = pooling["data"][0]["data"]
|
||||
assert isinstance(embeddings, list)
|
||||
assert len(embeddings) > 0
|
||||
assert len(embeddings[0]) == COLBERT_DIM
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_embed_not_supported(server: RemoteOpenAIServer):
|
||||
"""Test that ColBERT model does not support 'embed' task."""
|
||||
task = "embed"
|
||||
text = "What is the capital of France?"
|
||||
|
||||
response = requests.post(
|
||||
server.url_for("pooling"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"input": text,
|
||||
"task": task,
|
||||
},
|
||||
)
|
||||
|
||||
assert response.json()["error"]["type"] == "BadRequestError"
|
||||
assert response.json()["error"]["message"].startswith(f"Unsupported task: {task!r}")
|
||||
@@ -1,193 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from tests.entrypoints.pooling.scoring.util import (
|
||||
make_base64_image,
|
||||
make_image_mm_param,
|
||||
)
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.entrypoints.pooling.scoring.protocol import RerankResponse, ScoreResponse
|
||||
|
||||
MODEL_NAME = "vidore/colpali-v1.3-hf"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
with RemoteOpenAIServer(MODEL_NAME, []) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_score_api_query_text_vs_docs_image(server: RemoteOpenAIServer):
|
||||
query = "Describe the red object"
|
||||
|
||||
red_image = make_base64_image(64, 64, color=(255, 0, 0))
|
||||
blue_image = make_base64_image(64, 64, color=(0, 0, 255))
|
||||
|
||||
documents = [
|
||||
make_image_mm_param(red_image),
|
||||
make_image_mm_param(blue_image),
|
||||
]
|
||||
|
||||
score_response = requests.post(
|
||||
server.url_for("score"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"queries": query,
|
||||
"documents": documents,
|
||||
},
|
||||
)
|
||||
score_response.raise_for_status()
|
||||
scores = ScoreResponse.model_validate(score_response.json())
|
||||
|
||||
assert scores.id is not None
|
||||
assert scores.data is not None
|
||||
assert len(scores.data) == 2
|
||||
assert scores.data[0].score > scores.data[1].score
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_score_api_query_text_vs_docs_mix(server: RemoteOpenAIServer):
|
||||
red_image = make_base64_image(64, 64, color=(255, 0, 0))
|
||||
query = "What is the capital of France?"
|
||||
documents: list = [
|
||||
"The capital of France is Paris.",
|
||||
make_image_mm_param(red_image),
|
||||
]
|
||||
|
||||
score_response = requests.post(
|
||||
server.url_for("score"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"queries": query,
|
||||
"documents": documents,
|
||||
},
|
||||
)
|
||||
score_response.raise_for_status()
|
||||
scores = ScoreResponse.model_validate(score_response.json())
|
||||
|
||||
assert scores.id is not None
|
||||
assert scores.data is not None
|
||||
assert len(scores.data) == 2
|
||||
assert scores.data[0].score > scores.data[1].score
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_score_api_query_image_vs_docs_text(server: RemoteOpenAIServer):
|
||||
red_image = make_base64_image(64, 64, color=(255, 0, 0))
|
||||
image_query = make_image_mm_param(red_image, text="red color")
|
||||
|
||||
documents = [
|
||||
"Describe the red object.",
|
||||
"The capital of France is Paris.",
|
||||
]
|
||||
|
||||
score_response = requests.post(
|
||||
server.url_for("score"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"queries": image_query,
|
||||
"documents": documents,
|
||||
},
|
||||
)
|
||||
score_response.raise_for_status()
|
||||
scores = ScoreResponse.model_validate(score_response.json())
|
||||
|
||||
assert scores.id is not None
|
||||
assert scores.data is not None
|
||||
assert len(scores.data) == 2
|
||||
assert scores.data[0].score > scores.data[1].score
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rerank_api_query_text_vs_docs_image(server: RemoteOpenAIServer):
|
||||
query = "Describe the red object"
|
||||
|
||||
red_image = make_base64_image(64, 64, color=(255, 0, 0))
|
||||
blue_image = make_base64_image(64, 64, color=(0, 0, 255))
|
||||
|
||||
documents = [
|
||||
make_image_mm_param(red_image),
|
||||
make_image_mm_param(blue_image),
|
||||
]
|
||||
|
||||
rerank_response = requests.post(
|
||||
server.url_for("rerank"),
|
||||
json={"model": MODEL_NAME, "query": query, "documents": documents},
|
||||
)
|
||||
|
||||
rerank_response.raise_for_status()
|
||||
rerank = RerankResponse.model_validate(rerank_response.json())
|
||||
|
||||
assert rerank.id is not None
|
||||
assert rerank.results is not None
|
||||
assert len(rerank.results) == 2
|
||||
|
||||
red_result = next(r for r in rerank.results if r.index == 0)
|
||||
blue_result = next(r for r in rerank.results if r.index == 1)
|
||||
|
||||
assert red_result.relevance_score > blue_result.relevance_score
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rerank_api_query_text_vs_docs_mix(server: RemoteOpenAIServer):
|
||||
red_image = make_base64_image(64, 64, color=(255, 0, 0))
|
||||
query = "What is the capital of France?"
|
||||
documents: list = [
|
||||
"The capital of France is Paris.",
|
||||
make_image_mm_param(red_image),
|
||||
]
|
||||
|
||||
rerank_response = requests.post(
|
||||
server.url_for("rerank"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"query": query,
|
||||
"documents": documents,
|
||||
},
|
||||
)
|
||||
rerank_response.raise_for_status()
|
||||
rerank = RerankResponse.model_validate(rerank_response.json())
|
||||
|
||||
assert rerank.id is not None
|
||||
assert rerank.results is not None
|
||||
assert len(rerank.results) == 2
|
||||
|
||||
result0 = next(r for r in rerank.results if r.index == 0)
|
||||
result1 = next(r for r in rerank.results if r.index == 1)
|
||||
|
||||
assert result0.relevance_score > result1.relevance_score
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rerank_api_query_image_vs_docs_text(server: RemoteOpenAIServer):
|
||||
red_image = make_base64_image(64, 64, color=(255, 0, 0))
|
||||
image_query = make_image_mm_param(red_image, text="red color")
|
||||
|
||||
documents = [
|
||||
"Describe the red object.",
|
||||
"The capital of France is Paris.",
|
||||
]
|
||||
|
||||
rerank_response = requests.post(
|
||||
server.url_for("rerank"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"query": image_query,
|
||||
"documents": documents,
|
||||
},
|
||||
)
|
||||
rerank_response.raise_for_status()
|
||||
rerank = RerankResponse.model_validate(rerank_response.json())
|
||||
|
||||
assert rerank.id is not None
|
||||
assert rerank.results is not None
|
||||
assert len(rerank.results) == 2
|
||||
|
||||
result0 = next(r for r in rerank.results if r.index == 0)
|
||||
result1 = next(r for r in rerank.results if r.index == 1)
|
||||
|
||||
assert result0.relevance_score > result1.relevance_score
|
||||
@@ -1,141 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from unittest.mock import AsyncMock, Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.pooling.scoring.serving import ServingScores
|
||||
from vllm.entrypoints.pooling.typing import PoolingEngineInput, PoolingServeContext
|
||||
from vllm.inputs import tokens_input
|
||||
from vllm.pooling_params import PoolingParams
|
||||
|
||||
|
||||
def _make_context() -> PoolingServeContext:
|
||||
return PoolingServeContext(
|
||||
request=Mock(),
|
||||
model_name="model",
|
||||
request_id="score-caller-controlled-id",
|
||||
pooling_params=PoolingParams(task="token_embed"),
|
||||
lora_request=None,
|
||||
priorities=None,
|
||||
prompt_extras=None,
|
||||
engine_inputs=[
|
||||
PoolingEngineInput(
|
||||
prompts=tokens_input(prompt_token_ids=[1]),
|
||||
params=PoolingParams(task="token_embed"),
|
||||
lora_requests=None,
|
||||
priorities=0,
|
||||
),
|
||||
PoolingEngineInput(
|
||||
prompts=tokens_input(prompt_token_ids=[2]),
|
||||
params=PoolingParams(task="token_embed"),
|
||||
lora_requests=None,
|
||||
priorities=0,
|
||||
),
|
||||
],
|
||||
n_queries=1,
|
||||
)
|
||||
|
||||
|
||||
def _query_key(context: PoolingServeContext) -> str:
|
||||
assert context.engine_inputs is not None
|
||||
pooling_params = context.engine_inputs[0]["params"]
|
||||
late_interaction_params = pooling_params.late_interaction_params
|
||||
assert late_interaction_params is not None
|
||||
query_key = late_interaction_params.query_key
|
||||
assert query_key is not None
|
||||
return query_key
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skip_global_cleanup
|
||||
@pytest.mark.parametrize("failed_stage", ["query", "doc"])
|
||||
async def test_flash_late_interaction_cleans_up_queries_on_failure(
|
||||
failed_stage: str,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
serving = object.__new__(ServingScores)
|
||||
serving.io_processor = Mock()
|
||||
serving.engine_client = Mock(abort=AsyncMock(), collective_rpc=AsyncMock())
|
||||
|
||||
context = _make_context()
|
||||
monkeypatch.setattr(serving, "_init_ctx", AsyncMock(return_value=context))
|
||||
monkeypatch.setattr(serving, "_preprocessing", AsyncMock())
|
||||
|
||||
async def encode_queries(ctx):
|
||||
ctx.late_interaction_query_keys = ["query-key"]
|
||||
if failed_stage == "query":
|
||||
raise RuntimeError("query failed")
|
||||
|
||||
async def encode_docs(ctx):
|
||||
ctx.late_interaction_doc_keys = ["doc-key"]
|
||||
if failed_stage == "doc":
|
||||
raise RuntimeError("doc failed")
|
||||
|
||||
monkeypatch.setattr(
|
||||
serving,
|
||||
"_flash_late_interaction_encode_queries",
|
||||
AsyncMock(side_effect=encode_queries),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
serving,
|
||||
"_flash_late_interaction_encode_docs",
|
||||
AsyncMock(side_effect=encode_docs),
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match=f"{failed_stage} failed"):
|
||||
await serving.flash_late_interaction()
|
||||
|
||||
expected_doc_keys = [] if failed_stage == "query" else ["doc-key"]
|
||||
serving.engine_client.abort.assert_awaited_once_with(
|
||||
["query-key", *expected_doc_keys]
|
||||
)
|
||||
serving.engine_client.collective_rpc.assert_awaited_once_with(
|
||||
"release_late_interaction_query_cache",
|
||||
args=(["query-key"],),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_colliding_request_ids_use_distinct_late_interaction_keys(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
serving = object.__new__(ServingScores)
|
||||
serving.engine_client = Mock(abort=AsyncMock(), collective_rpc=AsyncMock())
|
||||
prepare_generators = AsyncMock()
|
||||
monkeypatch.setattr(serving, "_prepare_generators", prepare_generators)
|
||||
monkeypatch.setattr(serving, "_collect_batch", AsyncMock())
|
||||
|
||||
first = _make_context()
|
||||
second = _make_context()
|
||||
for context in (first, second):
|
||||
await serving._flash_late_interaction_encode_queries(context)
|
||||
await serving._flash_late_interaction_encode_docs(context)
|
||||
|
||||
prepared_contexts = [call.args[0] for call in prepare_generators.await_args_list]
|
||||
first_query_key = _query_key(prepared_contexts[0])
|
||||
first_doc_key = _query_key(prepared_contexts[1])
|
||||
second_query_key = _query_key(prepared_contexts[2])
|
||||
second_doc_key = _query_key(prepared_contexts[3])
|
||||
first_doc_request_ids = prepared_contexts[1].prompt_request_ids
|
||||
second_doc_request_ids = prepared_contexts[3].prompt_request_ids
|
||||
|
||||
assert first_query_key == first_doc_key
|
||||
assert second_query_key == second_doc_key
|
||||
assert first_query_key != second_query_key
|
||||
assert "caller-controlled-id" not in first_query_key
|
||||
assert "caller-controlled-id" not in second_query_key
|
||||
assert first_doc_request_ids is not None
|
||||
assert second_doc_request_ids is not None
|
||||
assert first.late_interaction_doc_keys == first_doc_request_ids
|
||||
assert second.late_interaction_doc_keys == second_doc_request_ids
|
||||
assert first_doc_request_ids != second_doc_request_ids
|
||||
assert all("caller-controlled-id" not in key for key in first_doc_request_ids)
|
||||
assert all("caller-controlled-id" not in key for key in second_doc_request_ids)
|
||||
|
||||
assert first.late_interaction_query_keys is not None
|
||||
await serving._cleanup_flash_late_interaction(first)
|
||||
serving.engine_client.abort.assert_awaited_once_with(
|
||||
[*first.late_interaction_query_keys, *first_doc_request_ids]
|
||||
)
|
||||
@@ -1,84 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
from tokenizers import Tokenizer, models, pre_tokenizers
|
||||
from transformers import TokenizersBackend
|
||||
|
||||
from vllm.entrypoints.pooling.scoring.typing import ScoreInput
|
||||
from vllm.entrypoints.pooling.scoring.utils import (
|
||||
truncate_text_to_tokens,
|
||||
validate_score_input,
|
||||
)
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("data_1", "data_2", "is_multimodal_model", "architecture", "message"),
|
||||
[
|
||||
pytest.param(
|
||||
{"content": []},
|
||||
"document",
|
||||
False,
|
||||
"TestModel",
|
||||
"MultiModalParam is not supported for TestModel",
|
||||
id="unsupported-multimodal-input",
|
||||
),
|
||||
pytest.param(
|
||||
["query 1", "query 2"],
|
||||
["document"],
|
||||
False,
|
||||
"TestModel",
|
||||
"Input lengths must be either 1:1, 1:N or N:N",
|
||||
id="incompatible-input-lengths",
|
||||
),
|
||||
pytest.param(
|
||||
[],
|
||||
["document"],
|
||||
False,
|
||||
"TestModel",
|
||||
"At least one text element must be given",
|
||||
id="empty-first-input",
|
||||
),
|
||||
pytest.param(
|
||||
["query"],
|
||||
[],
|
||||
False,
|
||||
"TestModel",
|
||||
"At least one text_pair element must be given",
|
||||
id="empty-second-input",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_validate_score_input_rejects_invalid_inputs(
|
||||
data_1: ScoreInput | list[ScoreInput],
|
||||
data_2: ScoreInput | list[ScoreInput],
|
||||
is_multimodal_model: bool,
|
||||
architecture: str,
|
||||
message: str,
|
||||
):
|
||||
with pytest.raises(VLLMValidationError) as exc_info:
|
||||
validate_score_input(
|
||||
data_1,
|
||||
data_2,
|
||||
is_multimodal_model=is_multimodal_model,
|
||||
architecture=architecture,
|
||||
)
|
||||
|
||||
assert str(exc_info.value) == message
|
||||
assert exc_info.value.parameter is None
|
||||
assert exc_info.value.value is None
|
||||
|
||||
|
||||
def test_truncate_text_to_tokens_handles_shared_character_offsets():
|
||||
vocab = {
|
||||
token: i for i, token in enumerate(sorted(pre_tokenizers.ByteLevel.alphabet()))
|
||||
}
|
||||
backend = Tokenizer(models.BPE(vocab=vocab, merges=[]))
|
||||
backend.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False)
|
||||
tokenizer = TokenizersBackend(tokenizer_object=backend)
|
||||
|
||||
truncated = truncate_text_to_tokens("aéx", tokenizer, max_tokens=2)
|
||||
|
||||
assert truncated == "a"
|
||||
assert len(tokenizer(truncated, add_special_tokens=False)["input_ids"]) == 1
|
||||
@@ -1,107 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from io import BytesIO
|
||||
|
||||
import pybase64 as base64
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from PIL import Image
|
||||
from safetensors.torch import load_file
|
||||
from transformers import AutoModel, AutoTokenizer
|
||||
|
||||
from tests.conftest import HfRunner
|
||||
from vllm.entrypoints.pooling.scoring.typing import ScoreMultiModalParam
|
||||
from vllm.entrypoints.pooling.scoring.utils import compute_maxsim_score
|
||||
from vllm.renderers.chat_utils import (
|
||||
ChatCompletionContentPartImageParam,
|
||||
ChatCompletionContentPartTextParam,
|
||||
)
|
||||
from vllm.transformers_utils.repo_utils import hf_api
|
||||
|
||||
|
||||
class ColBERTScoringHfRunner(torch.nn.Module):
|
||||
def __init__(self, model_name, linear_weights_key):
|
||||
super().__init__()
|
||||
|
||||
self.tokenizer = AutoTokenizer.from_pretrained(model_name)
|
||||
|
||||
self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
||||
|
||||
extra = {}
|
||||
if self.device.type == "cpu":
|
||||
extra["attn_implementation"] = "eager"
|
||||
|
||||
self.model = AutoModel.from_pretrained(
|
||||
model_name,
|
||||
**extra,
|
||||
).to(self.device)
|
||||
self.model.eval()
|
||||
|
||||
path = hf_api().hf_hub_download(model_name, filename="model.safetensors")
|
||||
weights = load_file(path)
|
||||
|
||||
self.linear_weight = weights[linear_weights_key].to(self.device).float()
|
||||
|
||||
@torch.inference_mode()
|
||||
def forward(self, texts):
|
||||
embeddings = []
|
||||
for text in texts:
|
||||
inputs = self.tokenizer(text, return_tensors="pt").to(self.device)
|
||||
hidden = self.model(**inputs).last_hidden_state.float()
|
||||
projected = F.linear(hidden, self.linear_weight.float())
|
||||
normalised = F.normalize(projected, p=2, dim=-1)
|
||||
embeddings.append(normalised.squeeze(0).cpu())
|
||||
return embeddings
|
||||
|
||||
@torch.inference_mode()
|
||||
def predict(self, prompts: list[list[str]], *args, **kwargs):
|
||||
hf_embeddings = [self(prompt) for prompt in prompts]
|
||||
hf_outputs = [
|
||||
compute_maxsim_score(*map(torch.tensor, pair)).item()
|
||||
for pair in hf_embeddings
|
||||
]
|
||||
return torch.as_tensor(hf_outputs)
|
||||
|
||||
|
||||
class EncoderScoringHfRunner(HfRunner):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs, is_sentence_transformer=True)
|
||||
|
||||
@torch.inference_mode()
|
||||
def predict(self, prompts: list[list[str]], *args, **kwargs):
|
||||
hf_embeddings = [self.encode(prompt) for prompt in prompts]
|
||||
hf_outputs = [
|
||||
F.cosine_similarity(*map(torch.tensor, pair), dim=0)
|
||||
for pair in hf_embeddings
|
||||
]
|
||||
return torch.as_tensor(hf_outputs)
|
||||
|
||||
|
||||
def make_base64_image(
|
||||
width: int = 64, height: int = 64, color: tuple[int, int, int] = (255, 0, 0)
|
||||
) -> str:
|
||||
"""Create a small solid-color PNG image and return its base64 data URI."""
|
||||
img = Image.new("RGB", (width, height), color)
|
||||
buf = BytesIO()
|
||||
img.save(buf, format="PNG")
|
||||
b64 = base64.b64encode(buf.getvalue()).decode()
|
||||
return f"data:image/png;base64,{b64}"
|
||||
|
||||
|
||||
def make_image_mm_param(
|
||||
image_uri: str,
|
||||
text: str | None = None,
|
||||
) -> ScoreMultiModalParam:
|
||||
"""Build a ScoreMultiModalParam containing an image (and optional text)."""
|
||||
content: list = [
|
||||
ChatCompletionContentPartImageParam(
|
||||
type="image_url",
|
||||
image_url={"url": image_uri},
|
||||
),
|
||||
]
|
||||
if text is not None:
|
||||
content.append(
|
||||
ChatCompletionContentPartTextParam(type="text", text=text),
|
||||
)
|
||||
return ScoreMultiModalParam(content=content)
|
||||
@@ -1,94 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.pooling import factories
|
||||
from vllm.entrypoints.pooling.factories import init_pooling_io_processors
|
||||
from vllm.entrypoints.pooling.pooling.io_processor import (
|
||||
PluginWithIOProcessorPlugins,
|
||||
UnsupportedCombinedTaskIOProcessor,
|
||||
)
|
||||
|
||||
|
||||
def _bge_m3_config(io_processor_plugin=None):
|
||||
model_config = MagicMock()
|
||||
model_config.get_pooling_task.return_value = "embed&token_classify"
|
||||
model_config.io_processor_plugin = io_processor_plugin
|
||||
model_config.hf_config.to_dict.return_value = {}
|
||||
model_config.architecture = "BgeM3EmbeddingModel"
|
||||
|
||||
vllm_config = MagicMock(model_config=model_config)
|
||||
renderer = MagicMock()
|
||||
renderer._executor = MagicMock()
|
||||
chat_template_config = MagicMock(
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
trust_request_chat_template=False,
|
||||
)
|
||||
return vllm_config, renderer, chat_template_config
|
||||
|
||||
|
||||
def test_combined_task_without_plugin_uses_rejection_processor():
|
||||
vllm_config, renderer, chat_template_config = _bge_m3_config()
|
||||
|
||||
processors = init_pooling_io_processors(
|
||||
supported_tasks=("embed", "embed&token_classify"),
|
||||
vllm_config=vllm_config,
|
||||
renderer=renderer,
|
||||
chat_template_config=chat_template_config,
|
||||
)
|
||||
|
||||
assert processors.keys() == {"embed&token_classify"}
|
||||
assert isinstance(
|
||||
processors["embed&token_classify"], UnsupportedCombinedTaskIOProcessor
|
||||
)
|
||||
|
||||
|
||||
def test_combined_task_with_plugin_uses_plugin_processor(monkeypatch):
|
||||
vllm_config, renderer, chat_template_config = _bge_m3_config("bge_m3_sparse_plugin")
|
||||
monkeypatch.setattr(factories, "has_io_processor", lambda *_: True)
|
||||
monkeypatch.setattr(
|
||||
"vllm.entrypoints.pooling.pooling.io_processor.get_io_processor",
|
||||
lambda *_: MagicMock(),
|
||||
)
|
||||
|
||||
processors = init_pooling_io_processors(
|
||||
supported_tasks=("embed", "embed&token_classify"),
|
||||
vllm_config=vllm_config,
|
||||
renderer=renderer,
|
||||
chat_template_config=chat_template_config,
|
||||
)
|
||||
|
||||
assert processors.keys() == {"embed&token_classify", "plugin"}
|
||||
assert isinstance(processors["plugin"], PluginWithIOProcessorPlugins)
|
||||
|
||||
|
||||
def test_combined_task_plain_pooling_request_has_actionable_error(monkeypatch):
|
||||
from vllm.entrypoints.pooling.pooling.protocol import PoolingCompletionRequest
|
||||
from vllm.entrypoints.pooling.pooling.serving import ServingPooling
|
||||
|
||||
vllm_config, renderer, chat_template_config = _bge_m3_config("bge_m3_sparse_plugin")
|
||||
monkeypatch.setattr(factories, "has_io_processor", lambda *_: True)
|
||||
monkeypatch.setattr(
|
||||
"vllm.entrypoints.pooling.pooling.io_processor.get_io_processor",
|
||||
lambda *_: MagicMock(),
|
||||
)
|
||||
|
||||
engine_client = MagicMock(renderer=renderer, vllm_config=vllm_config)
|
||||
models = MagicMock(model_config=vllm_config.model_config)
|
||||
serving = ServingPooling(
|
||||
engine_client,
|
||||
models,
|
||||
supported_tasks=("embed", "embed&token_classify"),
|
||||
request_logger=None,
|
||||
chat_template_config=chat_template_config,
|
||||
)
|
||||
request = PoolingCompletionRequest(model="BAAI/bge-m3", input=["hola"])
|
||||
|
||||
assert serving.io_processors.keys() == {"embed&token_classify", "plugin"}
|
||||
io_processor = serving.get_io_processor(request)
|
||||
with pytest.raises(ValueError, match="plugin request with a 'data' field"):
|
||||
io_processor.create_pooling_params(request)
|
||||
@@ -1,260 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm import PoolingParams
|
||||
from vllm.entrypoints.pooling.base.io_processor import PoolingIOProcessor
|
||||
from vllm.entrypoints.pooling.offline import PoolingOfflineMixin
|
||||
from vllm.entrypoints.pooling.scoring.io_processor import CrossEncoderIOProcessor
|
||||
from vllm.entrypoints.pooling.typing import OfflineEncodeInputsContext
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.renderers import TokenizeParams
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def processor() -> PoolingIOProcessor:
|
||||
return object.__new__(PoolingIOProcessor)
|
||||
|
||||
|
||||
def test_rejects_untrusted_request_chat_template(processor: PoolingIOProcessor):
|
||||
with pytest.raises(VLLMValidationError) as exc_info:
|
||||
processor._validate_chat_template("template", None, False)
|
||||
|
||||
assert str(exc_info.value) == (
|
||||
"Chat template is passed with request, but "
|
||||
"--trust-request-chat-template is not set. "
|
||||
"Refused request with untrusted chat template."
|
||||
)
|
||||
assert exc_info.value.parameter is None
|
||||
assert exc_info.value.value is None
|
||||
|
||||
|
||||
def test_rejects_mismatched_pooling_params(processor: PoolingIOProcessor):
|
||||
with pytest.raises(VLLMValidationError) as exc_info:
|
||||
processor._params_to_seq([PoolingParams()], num_requests=2)
|
||||
|
||||
assert str(exc_info.value) == (
|
||||
"The lengths of prompts (2) and params (1) must be the same."
|
||||
)
|
||||
assert exc_info.value.parameter is None
|
||||
assert exc_info.value.value is None
|
||||
|
||||
|
||||
def test_rejects_mismatched_lora_requests(processor: PoolingIOProcessor):
|
||||
with pytest.raises(VLLMValidationError) as exc_info:
|
||||
processor._lora_request_to_seq([None], num_requests=2)
|
||||
|
||||
assert str(exc_info.value) == (
|
||||
"The lengths of prompts (2) and lora_request (1) must be the same."
|
||||
)
|
||||
assert exc_info.value.parameter is None
|
||||
assert exc_info.value.value is None
|
||||
|
||||
|
||||
def test_rejects_conflicting_pooling_task(processor: PoolingIOProcessor):
|
||||
processor.model_config = SimpleNamespace(is_encoder_decoder=False)
|
||||
processor.renderer = SimpleNamespace(
|
||||
default_cmpl_tok_params=TokenizeParams(max_total_tokens=None)
|
||||
)
|
||||
ctx = OfflineEncodeInputsContext(
|
||||
pooling_task="embed",
|
||||
tokenization_kwargs=None,
|
||||
lora_request=None,
|
||||
priorities=None,
|
||||
prompts=[[1]],
|
||||
pooling_params=PoolingParams(task="classify"),
|
||||
)
|
||||
|
||||
with pytest.raises(VLLMValidationError) as exc_info:
|
||||
processor.get_request_factory_offline(ctx)
|
||||
|
||||
assert str(exc_info.value) == (
|
||||
"You cannot overwrite param.task='classify' with pooling_task='embed'!"
|
||||
)
|
||||
assert exc_info.value.parameter is None
|
||||
assert exc_info.value.value is None
|
||||
|
||||
|
||||
def test_qwen3_reranker_warns_without_chat_template(monkeypatch):
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from vllm.entrypoints.pooling.scoring.io_processor import CrossEncoderIOProcessor
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.model_loader.get_model_cls", lambda *_: MagicMock()
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.models.interfaces.supports_score_template",
|
||||
lambda *_: False,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.entrypoints.pooling.scoring.io_processor.is_mistral_tokenizer",
|
||||
lambda *_: False,
|
||||
)
|
||||
|
||||
mock_logger = MagicMock()
|
||||
monkeypatch.setattr(
|
||||
"vllm.entrypoints.pooling.scoring.io_processor.logger",
|
||||
mock_logger,
|
||||
)
|
||||
|
||||
model_config = MagicMock()
|
||||
model_config.architecture = "Qwen3ForSequenceClassification"
|
||||
model_config.model = "Qwen/Qwen3-Reranker-0.6B"
|
||||
model_config.hf_config.is_original_qwen3_reranker = True
|
||||
model_config.use_sep_token = False
|
||||
model_config.is_multimodal_model = False
|
||||
model_config.max_model_len = 8192
|
||||
|
||||
vllm_config = MagicMock(model_config=model_config)
|
||||
tokenizer = MagicMock()
|
||||
tokenizer.pad_token_id = 0
|
||||
renderer = MagicMock()
|
||||
renderer.get_tokenizer.return_value = tokenizer
|
||||
chat_template_config = MagicMock(chat_template=None)
|
||||
|
||||
CrossEncoderIOProcessor(
|
||||
vllm_config=vllm_config,
|
||||
renderer=renderer,
|
||||
chat_template_config=chat_template_config,
|
||||
)
|
||||
|
||||
mock_logger.warning.assert_called_once()
|
||||
warning_call = mock_logger.warning.call_args
|
||||
assert "Qwen/Qwen3-Reranker-0.6B" in warning_call[0][1]
|
||||
assert "qwen3_reranker.jinja" in warning_call[0][2]
|
||||
|
||||
|
||||
def test_qwen3_vl_reranker_warns_with_vl_template(monkeypatch):
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from vllm.entrypoints.pooling.scoring.io_processor import CrossEncoderIOProcessor
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.model_loader.get_model_cls", lambda *_: MagicMock()
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.models.interfaces.supports_score_template",
|
||||
lambda *_: False,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.entrypoints.pooling.scoring.io_processor.is_mistral_tokenizer",
|
||||
lambda *_: False,
|
||||
)
|
||||
|
||||
mock_logger = MagicMock()
|
||||
monkeypatch.setattr(
|
||||
"vllm.entrypoints.pooling.scoring.io_processor.logger",
|
||||
mock_logger,
|
||||
)
|
||||
|
||||
model_config = MagicMock()
|
||||
model_config.architecture = "Qwen3VLForSequenceClassification"
|
||||
model_config.model = "Qwen/Qwen3-VL-Reranker-2B"
|
||||
model_config.hf_config.is_original_qwen3_reranker = True
|
||||
model_config.use_sep_token = False
|
||||
model_config.is_multimodal_model = True
|
||||
model_config.max_model_len = 8192
|
||||
|
||||
vllm_config = MagicMock(model_config=model_config)
|
||||
tokenizer = MagicMock()
|
||||
tokenizer.pad_token_id = 0
|
||||
renderer = MagicMock()
|
||||
renderer.get_tokenizer.return_value = tokenizer
|
||||
chat_template_config = MagicMock(chat_template=None)
|
||||
|
||||
CrossEncoderIOProcessor(
|
||||
vllm_config=vllm_config,
|
||||
renderer=renderer,
|
||||
chat_template_config=chat_template_config,
|
||||
)
|
||||
|
||||
mock_logger.warning.assert_called_once()
|
||||
warning_call = mock_logger.warning.call_args
|
||||
assert "Qwen/Qwen3-VL-Reranker-2B" in warning_call[0][1]
|
||||
assert "qwen3_vl_reranker.jinja" in warning_call[0][2]
|
||||
|
||||
|
||||
def test_qwen3_reranker_no_warning_when_template_provided(monkeypatch):
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from vllm.entrypoints.pooling.scoring.io_processor import CrossEncoderIOProcessor
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.model_loader.get_model_cls", lambda *_: MagicMock()
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.models.interfaces.supports_score_template",
|
||||
lambda *_: False,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.entrypoints.pooling.scoring.io_processor.is_mistral_tokenizer",
|
||||
lambda *_: False,
|
||||
)
|
||||
|
||||
mock_logger = MagicMock()
|
||||
monkeypatch.setattr(
|
||||
"vllm.entrypoints.pooling.scoring.io_processor.logger",
|
||||
mock_logger,
|
||||
)
|
||||
|
||||
model_config = MagicMock()
|
||||
model_config.architecture = "Qwen3ForSequenceClassification"
|
||||
model_config.model = "Qwen/Qwen3-Reranker-0.6B"
|
||||
model_config.hf_config.is_original_qwen3_reranker = True
|
||||
model_config.use_sep_token = False
|
||||
model_config.is_multimodal_model = False
|
||||
model_config.max_model_len = 8192
|
||||
|
||||
vllm_config = MagicMock(model_config=model_config)
|
||||
tokenizer = MagicMock()
|
||||
tokenizer.pad_token_id = 0
|
||||
renderer = MagicMock()
|
||||
renderer.get_tokenizer.return_value = tokenizer
|
||||
chat_template_config = MagicMock(chat_template="custom template")
|
||||
|
||||
CrossEncoderIOProcessor(
|
||||
vllm_config=vllm_config,
|
||||
renderer=renderer,
|
||||
chat_template_config=chat_template_config,
|
||||
)
|
||||
|
||||
mock_logger.warning.assert_not_called()
|
||||
|
||||
|
||||
def _make_score_llm(monkeypatch) -> PoolingOfflineMixin:
|
||||
"""A scoring LLM with no model behind it.
|
||||
|
||||
Input validation runs for real; only the rendering, engine and
|
||||
post-processing boundaries are stubbed out. The pooling task is filled in
|
||||
before any of them, so the mutation is observable without inference.
|
||||
"""
|
||||
proc = CrossEncoderIOProcessor.__new__(CrossEncoderIOProcessor)
|
||||
proc.is_multimodal_model = False
|
||||
proc.architecture = "CrossEncoder"
|
||||
monkeypatch.setattr(proc, "get_request_factory_offline", lambda ctx: (None, 0))
|
||||
monkeypatch.setattr(proc, "post_process_offline", lambda ctx: [])
|
||||
|
||||
llm = PoolingOfflineMixin.__new__(PoolingOfflineMixin)
|
||||
llm.runner_type = "pooling"
|
||||
llm.pooling_task = "classify" # SCORE_TYPE_MAP -> "cross-encoder"
|
||||
llm.model_config = SimpleNamespace(hf_config=SimpleNamespace(num_labels=1))
|
||||
llm.pooling_io_processors = {"cross-encoder": proc}
|
||||
monkeypatch.setattr(llm, "_run_tiling_engine", lambda *args, **kwargs: [])
|
||||
return llm
|
||||
|
||||
|
||||
def test_score_leaves_caller_pooling_params_untouched(monkeypatch):
|
||||
llm = _make_score_llm(monkeypatch)
|
||||
|
||||
params = PoolingParams()
|
||||
llm.score("query", "doc", pooling_params=params)
|
||||
|
||||
assert params.task is None, (
|
||||
"LLM.score() wrote the model's own pooling task into the caller's "
|
||||
"PoolingParams; reusing that object with another pooling model now "
|
||||
"fails with 'You cannot overwrite ...' before any inference runs."
|
||||
)
|
||||
@@ -1,314 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest import mock
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm import PoolingParams
|
||||
from vllm.config import PoolerConfig
|
||||
from vllm.entrypoints.pooling.embed.io_processor import (
|
||||
JinaRankingTokenEmbedIOProcessor,
|
||||
TokenEmbedIOProcessor,
|
||||
)
|
||||
from vllm.entrypoints.pooling.late_chunking import build_late_chunking_metadata
|
||||
from vllm.entrypoints.pooling.offline import PoolingOfflineMixin
|
||||
from vllm.entrypoints.pooling.typing import (
|
||||
OfflineEncodeInputsContext,
|
||||
OfflineOutputsContext,
|
||||
)
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.outputs import (
|
||||
LateChunk,
|
||||
PoolingOutput,
|
||||
PoolingRequestOutput,
|
||||
RequestError,
|
||||
)
|
||||
from vllm.pooling_params import LateChunkingParams
|
||||
from vllm.renderers import TokenizeParams
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def late_chunk_processor():
|
||||
processor = object.__new__(TokenEmbedIOProcessor)
|
||||
processor.model_config = SimpleNamespace(
|
||||
is_encoder_decoder=False,
|
||||
architecture="NomicBertModel",
|
||||
model_impl="auto",
|
||||
is_matryoshka=False,
|
||||
hf_config=SimpleNamespace(),
|
||||
pooler_config=PoolerConfig(seq_pooling_type="MEAN", tok_pooling_type="ALL"),
|
||||
)
|
||||
processor.vllm_config = SimpleNamespace(
|
||||
model_config=processor.model_config,
|
||||
cache_config=SimpleNamespace(enable_prefix_caching=False),
|
||||
scheduler_config=SimpleNamespace(enable_chunked_prefill=False),
|
||||
lora_config=None,
|
||||
)
|
||||
processor.renderer = Mock()
|
||||
processor.renderer.default_cmpl_tok_params = TokenizeParams(max_total_tokens=32)
|
||||
processor.renderer.render_cmpl.side_effect = lambda **_: [
|
||||
{
|
||||
"type": "token",
|
||||
"prompt_token_ids": [101, 1, 2, 102],
|
||||
"prompt_token_offsets": [(0, 0), (0, 1), (2, 3), (0, 0)],
|
||||
}
|
||||
]
|
||||
return processor
|
||||
|
||||
|
||||
def _late_chunk_context(prompts="a b", **kwargs):
|
||||
return OfflineEncodeInputsContext(
|
||||
pooling_task="token_embed",
|
||||
tokenization_kwargs=kwargs or None,
|
||||
lora_request=None,
|
||||
priorities=None,
|
||||
prompts=prompts,
|
||||
pooling_params=PoolingParams(
|
||||
late_chunking_params=LateChunkingParams(chunk_size=2)
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def test_late_chunk_render_requests_offsets_once(late_chunk_processor):
|
||||
processor = late_chunk_processor
|
||||
ctx = _late_chunk_context()
|
||||
factory, count = processor.get_request_factory_offline(ctx)
|
||||
render_params = next(factory())
|
||||
result = processor.render(render_params)
|
||||
assert count == 1
|
||||
processor.renderer.render_cmpl.assert_called_once()
|
||||
assert processor.renderer.render_cmpl.call_args.kwargs[
|
||||
"tok_params"
|
||||
].return_token_offsets
|
||||
assert not render_params["tok_params"].return_token_offsets
|
||||
assert "prompt_token_offsets" not in result["prompts"]
|
||||
assert "late_chunking" not in result
|
||||
assert [c.char_range for c in ctx.late_chunking[0].metadata.chunks] == [
|
||||
(0, 1),
|
||||
(2, 3),
|
||||
]
|
||||
assert ctx.late_chunking[0].metadata.input_tokens == 4
|
||||
|
||||
|
||||
@pytest.mark.parametrize("as_sequence", [False, True])
|
||||
@pytest.mark.parametrize("chunk_size", [None, 2])
|
||||
def test_jina_ranking_rejects_chunking_before_tokenization(
|
||||
late_chunk_processor, as_sequence, chunk_size
|
||||
):
|
||||
processor = object.__new__(JinaRankingTokenEmbedIOProcessor)
|
||||
processor.__dict__.update(late_chunk_processor.__dict__)
|
||||
ctx = _late_chunk_context(prompts=["document", "query"])
|
||||
params = PoolingParams(
|
||||
late_chunking_params=LateChunkingParams(chunk_size) if chunk_size else None
|
||||
)
|
||||
ctx.pooling_params = [params] if as_sequence else params
|
||||
factory, count = processor.get_request_factory_offline(ctx)
|
||||
assert count == 1
|
||||
request = next(factory())
|
||||
if chunk_size is not None:
|
||||
with pytest.raises(VLLMValidationError, match="JinaForRanking.*late chunking"):
|
||||
processor.render(request)
|
||||
late_chunk_processor.renderer.render_cmpl.assert_not_called()
|
||||
else:
|
||||
result = processor.render(request)
|
||||
assert result["prompts"]["type"] == "token"
|
||||
assert result["prompts"]["prompt_token_ids"] == [101, 1, 2, 102]
|
||||
late_chunk_processor.renderer.render_cmpl.assert_called_once()
|
||||
assert not request["tok_params"].return_token_offsets
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"prompts, kwargs, error",
|
||||
[
|
||||
pytest.param("", {}, "plain-text", id="empty"),
|
||||
pytest.param([1, 2], {}, "plain-text", id="token-ids"),
|
||||
pytest.param(
|
||||
{"prompt": "abc", "multi_modal_data": {}},
|
||||
{},
|
||||
"plain-text",
|
||||
id="multimodal",
|
||||
),
|
||||
]
|
||||
+ [
|
||||
pytest.param("abc", {key: value}, "does not support", id=key)
|
||||
for key, value in (
|
||||
("truncate_prompt_tokens", 2),
|
||||
("truncation", True),
|
||||
("pad_prompt_tokens", 8),
|
||||
("padding", True),
|
||||
("do_lower_case", True),
|
||||
)
|
||||
],
|
||||
)
|
||||
def test_late_chunking_rejects_unsupported_input_before_render(
|
||||
late_chunk_processor, prompts, kwargs, error
|
||||
):
|
||||
processor = late_chunk_processor
|
||||
with pytest.raises(VLLMValidationError, match=error):
|
||||
factory, _ = processor.get_request_factory_offline(
|
||||
_late_chunk_context(prompts, **kwargs)
|
||||
)
|
||||
processor.render(next(factory()))
|
||||
processor.renderer.render_cmpl.assert_not_called()
|
||||
|
||||
|
||||
def test_late_chunk_ranges_keep_unicode_overlaps_and_special_only_chunks():
|
||||
text = "中 😀 e\u0301"
|
||||
offsets = [(0, 0), (0, 1), (2, 3), (2, 3), (4, 6), (0, 0)]
|
||||
metadata = build_late_chunking_metadata(text, len(offsets), offsets, 1)
|
||||
assert [c.char_range for c in metadata.chunks] == [
|
||||
None,
|
||||
(0, 1),
|
||||
(2, 3),
|
||||
(2, 3),
|
||||
(4, 6),
|
||||
None,
|
||||
]
|
||||
assert [
|
||||
text[slice(*c.char_range)] if c.char_range else None for c in metadata.chunks
|
||||
] == [None, "中", "😀", "😀", "e\u0301", None]
|
||||
chunks = build_late_chunking_metadata(text, len(offsets), offsets, 4).chunks
|
||||
assert [(c.token_range, c.char_range) for c in chunks] == [
|
||||
((0, 4), (0, 3)),
|
||||
((4, 6), (4, 6)),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"offsets", [None, [(0, 1)], [(-1, 1), (0, 0)], [(0, 4), (0, 0)], [(2, 3), (0, 1)]]
|
||||
)
|
||||
def test_late_chunk_ranges_reject_missing_or_invalid_offsets(offsets):
|
||||
with pytest.raises(VLLMValidationError, match="offsets"):
|
||||
build_late_chunking_metadata("abc", 2, offsets, 2)
|
||||
|
||||
|
||||
def _mock_chunk_llm(processor, outputs):
|
||||
llm = mock.Mock(spec=PoolingOfflineMixin)
|
||||
llm.encode = PoolingOfflineMixin.encode.__get__(llm)
|
||||
llm._run_tiling_engine = PoolingOfflineMixin._run_tiling_engine.__get__(llm)
|
||||
llm.pooling_io_processors = {"token_embed": processor}
|
||||
llm._executor = SimpleNamespace(map=map)
|
||||
llm.llm_engine = mock.Mock()
|
||||
llm.llm_engine.vllm_config.scheduler_config.max_num_seqs = 2
|
||||
llm.llm_engine.has_unfinished_requests.return_value = False
|
||||
llm.llm_engine.step.side_effect = outputs
|
||||
llm._render_and_add_requests = mock.Mock(
|
||||
side_effect=lambda **kw: [
|
||||
str(i) + "-internal" for i in range(len(kw["params"]))
|
||||
]
|
||||
)
|
||||
return llm
|
||||
|
||||
|
||||
def _chunk_output(request_id, rows=2, **kwargs):
|
||||
return PoolingRequestOutput(
|
||||
str(request_id),
|
||||
PoolingOutput(torch.ones(rows, 4)),
|
||||
[101, 1, 2, 102],
|
||||
0,
|
||||
True,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
def test_late_chunk_mapping_preserves_order_mixed_requests_and_errors(
|
||||
late_chunk_processor,
|
||||
):
|
||||
error = RequestError("test_error", "original error")
|
||||
failed = _chunk_output(3, rows=0, error=error)
|
||||
llm = _mock_chunk_llm(late_chunk_processor, RuntimeError("step failed"))
|
||||
params = [
|
||||
PoolingParams(late_chunking_params=LateChunkingParams(2)),
|
||||
PoolingParams(),
|
||||
PoolingParams(late_chunking_params=LateChunkingParams(1)),
|
||||
PoolingParams(late_chunking_params=LateChunkingParams(2)),
|
||||
]
|
||||
# A failed encode call must not leave metadata for the following batch.
|
||||
with pytest.raises(RuntimeError, match="step failed"):
|
||||
llm.encode(
|
||||
["a b"] * 2,
|
||||
pooling_task="token_embed",
|
||||
pooling_params=params[:2],
|
||||
use_tqdm=False,
|
||||
)
|
||||
llm.llm_engine.step.side_effect = [
|
||||
[
|
||||
_chunk_output(2, rows=4),
|
||||
failed,
|
||||
_chunk_output(1, rows=4),
|
||||
_chunk_output(0),
|
||||
]
|
||||
]
|
||||
outputs = llm.encode(
|
||||
["a b"] * 4, pooling_task="token_embed", pooling_params=params, use_tqdm=False
|
||||
)
|
||||
assert outputs[0].late_chunking.chunks == [
|
||||
LateChunk((0, 2), (0, 1)),
|
||||
LateChunk((2, 4), (2, 3)),
|
||||
]
|
||||
assert outputs[1].late_chunking is None
|
||||
assert len(outputs[2].late_chunking.chunks) == 4
|
||||
assert outputs[3].late_chunking is None
|
||||
assert outputs[3].error is error
|
||||
|
||||
|
||||
def test_late_chunk_mapping_rejects_successful_output_with_wrong_row_count(
|
||||
late_chunk_processor,
|
||||
):
|
||||
llm = _mock_chunk_llm(late_chunk_processor, [[_chunk_output(0, rows=0)]])
|
||||
with pytest.raises(ValueError, match="does not match"):
|
||||
llm.encode(
|
||||
"a b",
|
||||
pooling_task="token_embed",
|
||||
pooling_params=PoolingParams(late_chunking_params=LateChunkingParams(2)),
|
||||
use_tqdm=False,
|
||||
)
|
||||
# All engine requests have finished before post-processing validates the output.
|
||||
llm.llm_engine.abort_request.assert_not_called()
|
||||
|
||||
|
||||
def test_late_chunk_contexts_remain_isolated_when_rendering_interleaves(
|
||||
late_chunk_processor,
|
||||
):
|
||||
processor = late_chunk_processor
|
||||
first, second = _late_chunk_context("xy z"), _late_chunk_context()
|
||||
shared = PoolingParams(late_chunking_params=LateChunkingParams(2))
|
||||
first.pooling_params = second.pooling_params = shared
|
||||
first_factory, _ = processor.get_request_factory_offline(first)
|
||||
second_factory, _ = processor.get_request_factory_offline(second)
|
||||
first_request, second_request = next(first_factory()), next(second_factory())
|
||||
processor.render(second_request)
|
||||
assert first.late_chunking[0].metadata is None
|
||||
processor.renderer.render_cmpl.side_effect = lambda **_: [
|
||||
{
|
||||
"type": "token",
|
||||
"prompt_token_ids": [101, 1, 2, 102],
|
||||
"prompt_token_offsets": [(0, 0), (0, 2), (3, 4), (0, 0)],
|
||||
}
|
||||
]
|
||||
processor.render(first_request)
|
||||
first_output = processor.post_process_offline(
|
||||
OfflineOutputsContext([_chunk_output(0)], late_chunking=first.late_chunking)
|
||||
)[0]
|
||||
second_output = processor.post_process_offline(
|
||||
OfflineOutputsContext([_chunk_output(0)], late_chunking=second.late_chunking)
|
||||
)[0]
|
||||
assert first_output.late_chunking.chunks[0].char_range == (0, 2)
|
||||
assert second_output.late_chunking.chunks[0].char_range == (0, 1)
|
||||
|
||||
|
||||
def test_plain_token_embed_skips_chunk_metadata(late_chunk_processor):
|
||||
ctx = _late_chunk_context()
|
||||
ctx.pooling_params = PoolingParams()
|
||||
factory, _ = late_chunk_processor.get_request_factory_offline(ctx)
|
||||
request = next(factory())
|
||||
result = late_chunk_processor.render(request)
|
||||
assert not ctx.late_chunking
|
||||
assert result["params"].late_chunking_params is None
|
||||
assert not late_chunk_processor.renderer.render_cmpl.call_args.kwargs[
|
||||
"tok_params"
|
||||
].return_token_offsets
|
||||
@@ -1,59 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import importlib
|
||||
import importlib.util
|
||||
import json
|
||||
import warnings
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.entrypoints.pooling.utils import encode_pooling_output_float_or_ndarray
|
||||
|
||||
|
||||
def _pooling_output(data):
|
||||
return SimpleNamespace(outputs=SimpleNamespace(data=data))
|
||||
|
||||
|
||||
def test_encode_pooling_output_float_or_ndarray_returns_numpy_array():
|
||||
output = _pooling_output(torch.tensor([1.0, 2.0, 3.0], dtype=torch.float32))
|
||||
|
||||
encoded = encode_pooling_output_float_or_ndarray(output)
|
||||
|
||||
assert isinstance(encoded, np.ndarray)
|
||||
np.testing.assert_allclose(encoded, [1.0, 2.0, 3.0])
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
importlib.util.find_spec("orjson") is None,
|
||||
reason="orjson is not installed",
|
||||
)
|
||||
def test_orjson_serializes_numpy_array():
|
||||
from fastapi.responses import ORJSONResponse
|
||||
|
||||
output = _pooling_output(torch.tensor([1.0, 2.0, 3.0], dtype=torch.float32))
|
||||
encoded = encode_pooling_output_float_or_ndarray(output)
|
||||
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore", DeprecationWarning)
|
||||
response = ORJSONResponse(content={"embedding": encoded})
|
||||
assert json.loads(response.body)["embedding"] == pytest.approx([1.0, 2.0, 3.0])
|
||||
|
||||
|
||||
def test_encode_pooling_output_float_or_ndarray_falls_back_to_list():
|
||||
class DataWithUnsupportedNumpy:
|
||||
def is_contiguous(self):
|
||||
return True
|
||||
|
||||
def numpy(self):
|
||||
raise TypeError("unsupported dtype")
|
||||
|
||||
def tolist(self):
|
||||
return [1.0, 2.0, 3.0]
|
||||
|
||||
output = _pooling_output(DataWithUnsupportedNumpy())
|
||||
|
||||
assert encode_pooling_output_float_or_ndarray(output) == [1.0, 2.0, 3.0]
|
||||
@@ -1,73 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import weakref
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm import LLM, PoolingRequestOutput
|
||||
from vllm.config import PoolerConfig
|
||||
from vllm.tasks import PoolingTask
|
||||
|
||||
MODEL_NAME = "jason9693/Qwen2.5-1.5B-apeach"
|
||||
|
||||
prompt = "The chef prepared a delicious meal."
|
||||
prompt_token_ids = [785, 29706, 10030, 264, 17923, 15145, 13]
|
||||
num_labels = 2
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def llm(vllm_runner):
|
||||
with vllm_runner(
|
||||
MODEL_NAME,
|
||||
max_model_len=None,
|
||||
pooler_config=PoolerConfig(task="token_classify"),
|
||||
max_num_batched_tokens=32768,
|
||||
tensor_parallel_size=1,
|
||||
gpu_memory_utilization=0.75,
|
||||
enforce_eager=True,
|
||||
seed=0,
|
||||
enable_chunked_prefill=None,
|
||||
) as runner:
|
||||
# pytest caches yielded fixtures until after teardown, so use a proxy to
|
||||
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
|
||||
yield weakref.proxy(runner.llm)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_str_prompts(llm: LLM):
|
||||
outputs = llm.encode(prompt, pooling_task="token_classify", use_tqdm=False)
|
||||
assert len(outputs) == 1
|
||||
assert isinstance(outputs[0], PoolingRequestOutput)
|
||||
assert outputs[0].prompt_token_ids == prompt_token_ids
|
||||
assert outputs[0].outputs.data.shape == (len(prompt_token_ids), num_labels)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_token_ids_prompts(llm: LLM):
|
||||
outputs = llm.encode(
|
||||
[prompt_token_ids], pooling_task="token_classify", use_tqdm=False
|
||||
)
|
||||
assert len(outputs) == 1
|
||||
assert isinstance(outputs[0], PoolingRequestOutput)
|
||||
assert outputs[0].prompt_token_ids == prompt_token_ids
|
||||
assert outputs[0].outputs.data.shape == (len(prompt_token_ids), num_labels)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_score_api(llm: LLM):
|
||||
err_msg = "This model does not support the Scoring API."
|
||||
with pytest.raises(ValueError, match=err_msg):
|
||||
llm.score("ping", "pong", use_tqdm=False)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("task", ["classify", "embed", "token_embed", "plugin"])
|
||||
def test_unsupported_tasks(llm: LLM, task: PoolingTask, caplog_vllm):
|
||||
if task == "plugin":
|
||||
err_msg = "No IOProcessor plugin installed."
|
||||
elif task == "classify":
|
||||
err_msg = "Try switching the model's pooling_task via.+"
|
||||
else:
|
||||
err_msg = "Embedding API is not supported by this model.+"
|
||||
|
||||
with pytest.raises(ValueError, match=err_msg):
|
||||
llm.encode(prompt, pooling_task=task, use_tqdm=False)
|
||||
@@ -1,75 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.entrypoints.pooling.pooling.protocol import PoolingResponse
|
||||
|
||||
MODEL_NAME = "jason9693/Qwen2.5-1.5B-apeach"
|
||||
DTYPE = "float32" # Use float32 to avoid NaN issue
|
||||
input_text = "This product was excellent and exceeded my expectations"
|
||||
input_tokens = [1986, 1985, 572, 9073, 323, 33808, 847, 16665]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
args = [
|
||||
"--enforce-eager",
|
||||
"--max-model-len",
|
||||
"512",
|
||||
"--dtype",
|
||||
DTYPE,
|
||||
"--pooler-config.task",
|
||||
"token_classify",
|
||||
]
|
||||
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
async def test_pooling_token_classify(server: RemoteOpenAIServer, model_name: str):
|
||||
task = "token_classify"
|
||||
response = requests.post(
|
||||
server.url_for("pooling"),
|
||||
json={
|
||||
"model": model_name,
|
||||
"input": input_text,
|
||||
"encoding_format": "float",
|
||||
"task": task,
|
||||
},
|
||||
)
|
||||
poolings = PoolingResponse.model_validate(response.json())
|
||||
assert len(poolings.data) == 1
|
||||
assert len(poolings.data[0].data) == 8
|
||||
assert isinstance(poolings.data[0].data[0], list)
|
||||
assert len(poolings.data[0].data[0]) == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
@pytest.mark.parametrize("task", ["classify", "embed", "token_embed", "plugin"])
|
||||
async def test_pooling_not_supported(
|
||||
server: RemoteOpenAIServer, model_name: str, task: str
|
||||
):
|
||||
response = requests.post(
|
||||
server.url_for("pooling"),
|
||||
json={
|
||||
"model": model_name,
|
||||
"input": input_text,
|
||||
"encoding_format": "float",
|
||||
"task": task,
|
||||
},
|
||||
)
|
||||
assert response.json()["error"]["type"] == "BadRequestError"
|
||||
|
||||
if task == "plugin":
|
||||
err_msg = "No IOProcessor plugin installed."
|
||||
elif task == "classify":
|
||||
err_msg = "Try switching the model's pooling_task via"
|
||||
else:
|
||||
err_msg = f"Unsupported task: {task!r}"
|
||||
assert response.json()["error"]["message"].startswith(err_msg)
|
||||
@@ -1,282 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import weakref
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.models.utils import check_embeddings_close
|
||||
from vllm import LLM, PoolingParams, PoolingRequestOutput
|
||||
from vllm.config import PoolerConfig
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.pooling_params import LateChunkingParams
|
||||
from vllm.tasks import PoolingTask
|
||||
|
||||
MODEL_NAME = "intfloat/multilingual-e5-small"
|
||||
|
||||
prompt = "The chef prepared a delicious meal."
|
||||
prompt_token_ids = [0, 581, 21861, 133888, 10, 8, 150, 60744, 109911, 5, 2]
|
||||
embedding_size = 384
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def llm(vllm_runner):
|
||||
with vllm_runner(
|
||||
MODEL_NAME,
|
||||
max_model_len=None,
|
||||
pooler_config=PoolerConfig(task="token_embed"),
|
||||
max_num_batched_tokens=32768,
|
||||
tensor_parallel_size=1,
|
||||
gpu_memory_utilization=0.75,
|
||||
enforce_eager=True,
|
||||
seed=0,
|
||||
enable_chunked_prefill=None,
|
||||
) as runner:
|
||||
assert embedding_size == runner.llm.model_config.embedding_size
|
||||
# pytest caches yielded fixtures until after teardown, so use a proxy to
|
||||
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
|
||||
yield weakref.proxy(runner.llm)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_str_prompts(llm: LLM):
|
||||
outputs = llm.encode(prompt, pooling_task="token_embed", use_tqdm=False)
|
||||
assert len(outputs) == 1
|
||||
assert isinstance(outputs[0], PoolingRequestOutput)
|
||||
assert outputs[0].outputs.data.shape == (11, 384)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_token_ids_prompts(llm: LLM):
|
||||
outputs = llm.encode([prompt_token_ids], pooling_task="token_embed", use_tqdm=False)
|
||||
assert len(outputs) == 1
|
||||
assert isinstance(outputs[0], PoolingRequestOutput)
|
||||
assert outputs[0].outputs.data.shape == (11, 384)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("task", ["embed", "classify", "token_classify", "plugin"])
|
||||
def test_unsupported_tasks(llm: LLM, task: PoolingTask, caplog_vllm):
|
||||
if task == "plugin":
|
||||
err_msg = "No IOProcessor plugin installed."
|
||||
elif task == "embed":
|
||||
err_msg = "Try switching the model's pooling_task via.+"
|
||||
else:
|
||||
err_msg = "Classification API is not supported by this model.+"
|
||||
|
||||
with pytest.raises(ValueError, match=err_msg):
|
||||
llm.encode(prompt, pooling_task=task, use_tqdm=False)
|
||||
|
||||
|
||||
def test_bge_m3_late_chunking_keeps_special_tokens(vllm_runner):
|
||||
with vllm_runner(
|
||||
"BAAI/bge-m3",
|
||||
hf_overrides={"architectures": ["BgeM3EmbeddingModel"]},
|
||||
pooler_config=PoolerConfig(task="token_embed"),
|
||||
dtype="float32",
|
||||
max_model_len=128,
|
||||
# Keep encoder batch size fixed for the numerical reference.
|
||||
max_num_seqs=1,
|
||||
gpu_memory_utilization=0.5,
|
||||
enforce_eager=True,
|
||||
) as runner:
|
||||
llm = runner.llm
|
||||
text = "A document about Berlin and its museums."
|
||||
sizes = (1, 3, 128, None)
|
||||
tokens, *outputs = llm.encode(
|
||||
[text] * len(sizes),
|
||||
pooling_task="token_embed",
|
||||
pooling_params=[
|
||||
PoolingParams(
|
||||
use_activation=size not in (1, None),
|
||||
late_chunking_params=LateChunkingParams(size) if size else None,
|
||||
)
|
||||
for size in sizes
|
||||
],
|
||||
use_tqdm=False,
|
||||
)
|
||||
assert tokens.outputs.data.shape[0] == len(tokens.prompt_token_ids)
|
||||
assert tokens.late_chunking.chunks[0].char_range is None
|
||||
for size, output in zip(sizes[1:], outputs):
|
||||
if size is None:
|
||||
# Ordinary BGE-M3 token embeddings still drop BOS only.
|
||||
expected = tokens.outputs.data[1:]
|
||||
assert output.late_chunking is None
|
||||
else:
|
||||
# BGE-M3's projector is linear, so its unnormalized token
|
||||
# outputs also provide a reference for projected chunk means.
|
||||
expected = torch.nn.functional.normalize(
|
||||
torch.stack([p.mean(0) for p in tokens.outputs.data.split(size)]),
|
||||
dim=-1,
|
||||
)
|
||||
assert len(output.late_chunking.chunks) == len(expected)
|
||||
torch.testing.assert_close(
|
||||
output.outputs.data, expected, atol=2e-5, rtol=2e-4
|
||||
)
|
||||
|
||||
|
||||
def test_nomic_late_chunking_without_special_tokens(vllm_runner):
|
||||
"""Keep source ranges without special tokens and reject empty tokenization."""
|
||||
with vllm_runner(
|
||||
"nomic-ai/nomic-embed-text-v1",
|
||||
revision="720244025c1a7e15661a174c63cce63c8218e52b",
|
||||
trust_remote_code=True,
|
||||
runner="pooling",
|
||||
pooler_config=PoolerConfig(task="token_embed"),
|
||||
dtype="float32",
|
||||
max_model_len=128,
|
||||
max_num_seqs=2,
|
||||
gpu_memory_utilization=0.3,
|
||||
enforce_eager=True,
|
||||
enable_prefix_caching=False,
|
||||
enable_chunked_prefill=False,
|
||||
) as runner:
|
||||
llm = runner.llm
|
||||
text = "search_document: 中文 😀 café e\u0301"
|
||||
params = PoolingParams(
|
||||
late_chunking_params=LateChunkingParams(3), use_activation=False
|
||||
)
|
||||
kwargs = {"add_special_tokens": False}
|
||||
raw = llm.encode(
|
||||
text,
|
||||
pooling_task="token_embed",
|
||||
use_tqdm=False,
|
||||
pooling_params=PoolingParams(use_activation=False),
|
||||
tokenization_kwargs=kwargs,
|
||||
)[0]
|
||||
actual = llm.encode(
|
||||
text,
|
||||
pooling_task="token_embed",
|
||||
use_tqdm=False,
|
||||
pooling_params=params,
|
||||
tokenization_kwargs=kwargs,
|
||||
)[0]
|
||||
expected = torch.stack([part.mean(0) for part in raw.outputs.data.split(3)])
|
||||
torch.testing.assert_close(actual.outputs.data, expected, atol=1e-5, rtol=1e-4)
|
||||
encoded = llm.get_tokenizer()(text, return_offsets_mapping=True, **kwargs)
|
||||
assert actual.prompt_token_ids == encoded["input_ids"]
|
||||
for chunk in actual.late_chunking.chunks:
|
||||
start, end = chunk.token_range
|
||||
offsets = [(a, b) for a, b in encoded["offset_mapping"][start:end] if a < b]
|
||||
assert chunk.char_range == (offsets[0][0], offsets[-1][1])
|
||||
for rejected in ("long " * 256, " "):
|
||||
with pytest.raises(VLLMValidationError):
|
||||
llm.encode(
|
||||
rejected,
|
||||
pooling_task="token_embed",
|
||||
use_tqdm=False,
|
||||
pooling_params=params,
|
||||
tokenization_kwargs=kwargs,
|
||||
)
|
||||
assert (
|
||||
llm.encode(text, pooling_task="token_embed", use_tqdm=False)[
|
||||
0
|
||||
].late_chunking
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
@pytest.mark.parametrize("normalize", [False, True])
|
||||
def test_e5_late_chunking_mixed_requests(llm: LLM, normalize: bool):
|
||||
texts = [prompt, "Short text.", "中文 😀 café"]
|
||||
raw = llm.encode(
|
||||
texts,
|
||||
pooling_task="token_embed",
|
||||
use_tqdm=False,
|
||||
pooling_params=PoolingParams(use_activation=False),
|
||||
)
|
||||
sizes = [3, None, 100]
|
||||
params = [
|
||||
PoolingParams(
|
||||
late_chunking_params=LateChunkingParams(size) if size is not None else None,
|
||||
use_activation=normalize,
|
||||
)
|
||||
for size in sizes
|
||||
]
|
||||
actual = llm.encode(
|
||||
texts, pooling_task="token_embed", pooling_params=params, use_tqdm=False
|
||||
)
|
||||
for original, result, size in zip(raw, actual, sizes):
|
||||
expected = original.outputs.data
|
||||
if size is not None:
|
||||
expected = torch.stack(
|
||||
[part.float().mean(0) for part in expected.split(size)]
|
||||
)
|
||||
assert result.late_chunking is not None
|
||||
assert len(result.late_chunking.chunks) == len(expected)
|
||||
assert result.late_chunking.input_tokens == len(result.prompt_token_ids)
|
||||
else:
|
||||
assert result.late_chunking is None
|
||||
if normalize:
|
||||
expected = torch.nn.functional.normalize(expected, dim=-1)
|
||||
check_embeddings_close(
|
||||
embeddings_0_lst=result.outputs.data.tolist(),
|
||||
embeddings_1_lst=expected.tolist(),
|
||||
name_0="late_chunking",
|
||||
name_1="token_reference",
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
result.outputs.data.norm(dim=-1),
|
||||
expected.norm(dim=-1),
|
||||
atol=1e-3,
|
||||
rtol=1e-3,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_v2_runner", [False, True])
|
||||
def test_late_chunking_recomputes_cached_prefix_with_chunked_prefill(
|
||||
vllm_runner, monkeypatch, use_v2_runner
|
||||
):
|
||||
# Dummy weights suffice for testing scheduling and reduction of the same states.
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1" if use_v2_runner else "0")
|
||||
with vllm_runner(
|
||||
"openai-community/gpt2",
|
||||
runner="pooling",
|
||||
convert="embed",
|
||||
load_format="dummy",
|
||||
pooler_config=PoolerConfig(
|
||||
task="token_embed", seq_pooling_type="LAST", tok_pooling_type="ALL"
|
||||
),
|
||||
dtype="float32",
|
||||
max_model_len=128,
|
||||
max_num_batched_tokens=16,
|
||||
max_num_seqs=2,
|
||||
enable_prefix_caching=True,
|
||||
enable_chunked_prefill=True,
|
||||
enforce_eager=True,
|
||||
gpu_memory_utilization=0.3,
|
||||
seed=0,
|
||||
) as runner:
|
||||
text = "The document has several important words. " * 6
|
||||
llm = runner.llm
|
||||
raw = llm.encode(
|
||||
text,
|
||||
pooling_task="token_embed",
|
||||
use_tqdm=False,
|
||||
pooling_params=PoolingParams(use_activation=False),
|
||||
)[0]
|
||||
assert len(raw.prompt_token_ids) > 16
|
||||
cached = llm.encode(
|
||||
text,
|
||||
pooling_task="token_embed",
|
||||
use_tqdm=False,
|
||||
pooling_params=PoolingParams(
|
||||
use_activation=False, skip_reading_prefix_cache=False
|
||||
),
|
||||
)[0]
|
||||
assert cached.num_cached_tokens > 0
|
||||
params = PoolingParams(
|
||||
late_chunking_params=LateChunkingParams(7),
|
||||
use_activation=False,
|
||||
skip_reading_prefix_cache=False,
|
||||
)
|
||||
actual = llm.encode(
|
||||
text, pooling_task="token_embed", pooling_params=params, use_tqdm=False
|
||||
)[0]
|
||||
assert actual.num_cached_tokens == 0
|
||||
assert actual.prompt_token_ids == raw.prompt_token_ids
|
||||
expected = torch.stack([part.mean(0) for part in raw.outputs.data.split(7)])
|
||||
torch.testing.assert_close(actual.outputs.data, expected, atol=1e-5, rtol=1e-4)
|
||||
assert actual.late_chunking.input_tokens == len(raw.prompt_token_ids)
|
||||
assert params.skip_reading_prefix_cache is False
|
||||
assert params.late_chunking_params is not None
|
||||
@@ -1,101 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.entrypoints.pooling.pooling.protocol import PoolingResponse
|
||||
|
||||
MODEL_NAME = "intfloat/multilingual-e5-small"
|
||||
DTYPE = "bfloat16"
|
||||
input_text = "The best thing about vLLM is that it supports many different models"
|
||||
input_tokens = [
|
||||
0,
|
||||
581,
|
||||
2965,
|
||||
13580,
|
||||
1672,
|
||||
81,
|
||||
23708,
|
||||
594,
|
||||
83,
|
||||
450,
|
||||
442,
|
||||
8060,
|
||||
7,
|
||||
5941,
|
||||
12921,
|
||||
115774,
|
||||
2,
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
args = [
|
||||
"--runner",
|
||||
"pooling",
|
||||
"--dtype",
|
||||
DTYPE,
|
||||
"--enforce-eager",
|
||||
"--max-model-len",
|
||||
"512",
|
||||
"--pooler-config.task",
|
||||
"token_embed",
|
||||
]
|
||||
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
async def test_pooling_token_embed(server: RemoteOpenAIServer, model_name: str):
|
||||
task = "token_embed"
|
||||
request_id = "client-request-id"
|
||||
response = requests.post(
|
||||
server.url_for("pooling"),
|
||||
json={
|
||||
"model": model_name,
|
||||
"input": input_text,
|
||||
"encoding_format": "float",
|
||||
"task": task,
|
||||
"request_id": request_id,
|
||||
},
|
||||
)
|
||||
|
||||
poolings = PoolingResponse.model_validate(response.json())
|
||||
|
||||
assert poolings.id == f"pooling-{request_id}"
|
||||
assert len(poolings.data) == 1
|
||||
assert len(poolings.data[0].data) == len(input_tokens)
|
||||
assert isinstance(poolings.data[0].data[0], list)
|
||||
assert len(poolings.data[0].data[0]) == 384
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
@pytest.mark.parametrize("task", ["embed", "classify", "token_classify", "plugin"])
|
||||
async def test_pooling_not_supported(
|
||||
server: RemoteOpenAIServer, model_name: str, task: str
|
||||
):
|
||||
response = requests.post(
|
||||
server.url_for("pooling"),
|
||||
json={
|
||||
"model": model_name,
|
||||
"input": "test",
|
||||
"encoding_format": "float",
|
||||
"task": task,
|
||||
},
|
||||
)
|
||||
assert response.json()["error"]["type"] == "BadRequestError"
|
||||
|
||||
if task == "plugin":
|
||||
err_msg = "No IOProcessor plugin installed."
|
||||
elif task == "embed":
|
||||
err_msg = "Try switching the model's pooling_task via"
|
||||
else:
|
||||
err_msg = f"Unsupported task: {task!r}"
|
||||
assert response.json()["error"]["message"].startswith(err_msg)
|
||||
@@ -1,218 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from http import HTTPStatus
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm import PoolingParams
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.pooling.base.serving import PoolingBaseServing
|
||||
from vllm.entrypoints.pooling.typing import PoolingServeContext
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.lora.protocol import (
|
||||
LoadLoRAAdapterRequest,
|
||||
UnloadLoRAAdapterRequest,
|
||||
)
|
||||
from vllm.exceptions import VLLMNotFoundError
|
||||
from vllm.lora.request import LoRARequest
|
||||
|
||||
MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
|
||||
BASE_MODEL_PATHS = [BaseModelPath(name=MODEL_NAME, model_path=MODEL_NAME)]
|
||||
LORA_LOADING_SUCCESS_MESSAGE = "Success: LoRA adapter '{lora_name}' added successfully."
|
||||
LORA_UNLOADING_SUCCESS_MESSAGE = (
|
||||
"Success: LoRA adapter '{lora_name}' removed successfully."
|
||||
)
|
||||
|
||||
|
||||
async def _async_serving_models_init_with_mock() -> tuple[
|
||||
OpenAIServingModels, MagicMock
|
||||
]:
|
||||
mock_engine_client = MagicMock(spec=EngineClient)
|
||||
# Set the max_model_len attribute to avoid missing attribute
|
||||
mock_model_config = MagicMock(spec=ModelConfig)
|
||||
mock_model_config.max_model_len = 2048
|
||||
mock_engine_client.model_config = mock_model_config
|
||||
mock_engine_client.input_processor = MagicMock()
|
||||
mock_engine_client.renderer = MagicMock()
|
||||
|
||||
serving_models = OpenAIServingModels(
|
||||
engine_client=mock_engine_client,
|
||||
base_model_paths=BASE_MODEL_PATHS,
|
||||
lora_modules=None,
|
||||
)
|
||||
await serving_models.init_static_loras()
|
||||
|
||||
# The mock is returned separately: `engine_client` is typed as the protocol,
|
||||
# so assertion helpers are not visible through it.
|
||||
return serving_models, mock_engine_client
|
||||
|
||||
|
||||
async def _async_serving_models_init() -> OpenAIServingModels:
|
||||
serving_models, _ = await _async_serving_models_init_with_mock()
|
||||
return serving_models
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_serving_model_name():
|
||||
serving_models = await _async_serving_models_init()
|
||||
assert serving_models.model_name(None) == MODEL_NAME
|
||||
request = LoRARequest(
|
||||
lora_name="adapter", lora_path="/path/to/adapter2", lora_int_id=1
|
||||
)
|
||||
assert serving_models.model_name(request) == request.lora_name
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_load_lora_adapter_success():
|
||||
serving_models = await _async_serving_models_init()
|
||||
request = LoadLoRAAdapterRequest(lora_name="adapter", lora_path="/path/to/adapter2")
|
||||
response = await serving_models.load_lora_adapter(request)
|
||||
assert response == LORA_LOADING_SUCCESS_MESSAGE.format(lora_name="adapter")
|
||||
assert len(serving_models.lora_requests) == 1
|
||||
assert "adapter" in serving_models.lora_requests
|
||||
assert serving_models.lora_requests["adapter"].lora_name == "adapter"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_load_lora_adapter_missing_fields():
|
||||
serving_models = await _async_serving_models_init()
|
||||
request = LoadLoRAAdapterRequest(lora_name="", lora_path="")
|
||||
response = await serving_models.load_lora_adapter(request)
|
||||
assert isinstance(response, ErrorResponse)
|
||||
assert response.error.type == "InvalidUserInput"
|
||||
assert response.error.code == HTTPStatus.BAD_REQUEST
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_load_lora_adapter_duplicate():
|
||||
serving_models = await _async_serving_models_init()
|
||||
request = LoadLoRAAdapterRequest(
|
||||
lora_name="adapter1", lora_path="/path/to/adapter1"
|
||||
)
|
||||
response = await serving_models.load_lora_adapter(request)
|
||||
assert response == LORA_LOADING_SUCCESS_MESSAGE.format(lora_name="adapter1")
|
||||
assert len(serving_models.lora_requests) == 1
|
||||
|
||||
request = LoadLoRAAdapterRequest(
|
||||
lora_name="adapter1", lora_path="/path/to/adapter1"
|
||||
)
|
||||
response = await serving_models.load_lora_adapter(request)
|
||||
assert isinstance(response, ErrorResponse)
|
||||
assert response.error.type == "InvalidUserInput"
|
||||
assert response.error.code == HTTPStatus.BAD_REQUEST
|
||||
assert len(serving_models.lora_requests) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("load_inplace", [False, True])
|
||||
async def test_load_lora_adapter_base_model_name(load_inplace: bool):
|
||||
"""A LoRA named after a served model would shadow it for all requests."""
|
||||
serving_models, mock_engine_client = await _async_serving_models_init_with_mock()
|
||||
request = LoadLoRAAdapterRequest(
|
||||
lora_name=MODEL_NAME, lora_path="/path/to/adapter", load_inplace=load_inplace
|
||||
)
|
||||
response = await serving_models.load_lora_adapter(request)
|
||||
assert isinstance(response, ErrorResponse)
|
||||
assert response.error.type == "InvalidUserInput"
|
||||
assert response.error.code == HTTPStatus.BAD_REQUEST
|
||||
assert len(serving_models.lora_requests) == 0
|
||||
mock_engine_client.add_lora.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unload_lora_adapter_success():
|
||||
serving_models = await _async_serving_models_init()
|
||||
request = LoadLoRAAdapterRequest(
|
||||
lora_name="adapter1", lora_path="/path/to/adapter1"
|
||||
)
|
||||
response = await serving_models.load_lora_adapter(request)
|
||||
assert len(serving_models.lora_requests) == 1
|
||||
|
||||
unload_request = UnloadLoRAAdapterRequest(lora_name="adapter1")
|
||||
response = await serving_models.unload_lora_adapter(unload_request)
|
||||
assert response == LORA_UNLOADING_SUCCESS_MESSAGE.format(lora_name="adapter1")
|
||||
assert len(serving_models.lora_requests) == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unload_lora_adapter_missing_fields():
|
||||
serving_models = await _async_serving_models_init()
|
||||
request = UnloadLoRAAdapterRequest(lora_name="", lora_int_id=None)
|
||||
response = await serving_models.unload_lora_adapter(request)
|
||||
assert isinstance(response, ErrorResponse)
|
||||
assert response.error.type == "InvalidUserInput"
|
||||
assert response.error.code == HTTPStatus.BAD_REQUEST
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unload_lora_adapter_not_found():
|
||||
serving_models = await _async_serving_models_init()
|
||||
request = UnloadLoRAAdapterRequest(lora_name="nonexistent_adapter")
|
||||
response = await serving_models.unload_lora_adapter(request)
|
||||
assert isinstance(response, ErrorResponse)
|
||||
assert response.error.type == "NotFoundError"
|
||||
assert response.error.code == HTTPStatus.NOT_FOUND
|
||||
|
||||
|
||||
class _ConcretePoolingServing(PoolingBaseServing):
|
||||
"""Minimal concrete subclass used only in these unit tests."""
|
||||
|
||||
request_id_prefix = "test"
|
||||
|
||||
def get_io_processor(self, request):
|
||||
raise NotImplementedError
|
||||
|
||||
def _build_response(self, ctx):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
def _make_pooling_serving(lora_name: str) -> _ConcretePoolingServing:
|
||||
lora_request = LoRARequest(
|
||||
lora_name=lora_name, lora_int_id=1, lora_path="/path/to/lora"
|
||||
)
|
||||
mock_models = MagicMock()
|
||||
mock_models.lora_requests = {lora_name: lora_request}
|
||||
mock_models.is_base_model.side_effect = lambda name: name == MODEL_NAME
|
||||
|
||||
serving = object.__new__(_ConcretePoolingServing)
|
||||
serving.models = mock_models
|
||||
return serving
|
||||
|
||||
|
||||
def _make_pooling_ctx(
|
||||
model_name: str, serving: PoolingBaseServing
|
||||
) -> PoolingServeContext:
|
||||
mock_request = MagicMock()
|
||||
mock_request.model = model_name
|
||||
return PoolingServeContext(
|
||||
request=mock_request,
|
||||
model_name=MODEL_NAME,
|
||||
request_id="test-id",
|
||||
pooling_params=PoolingParams(),
|
||||
lora_request=serving._maybe_get_adapters(mock_request),
|
||||
priorities=0,
|
||||
prompt_extras=None,
|
||||
)
|
||||
|
||||
|
||||
def test_pooling_maybe_get_adapters_lora_name_sets_lora_request():
|
||||
"""LoRA adapter name must populate ctx.lora_request without raising."""
|
||||
lora_name = "bot-embed-lora"
|
||||
serving = _make_pooling_serving(lora_name)
|
||||
ctx = _make_pooling_ctx(lora_name, serving)
|
||||
|
||||
assert ctx.lora_request is not None
|
||||
assert ctx.lora_request.lora_name == lora_name
|
||||
|
||||
|
||||
def test_pooling_maybe_get_adapters_unknown_model_raises():
|
||||
"""An unrecognised model name must still raise VLLMNotFoundError."""
|
||||
serving = _make_pooling_serving("some-lora")
|
||||
|
||||
with pytest.raises(VLLMNotFoundError):
|
||||
_make_pooling_ctx("unknown-model", serving)
|
||||
@@ -1,402 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Evaluate Transcription API correctness by computing Word Error Rate (WER)
|
||||
on a given ASR dataset. When provided, it will also compare the WER against
|
||||
a baseline.
|
||||
This simulates real work usage of the API and makes sure that the frontend and
|
||||
AsyncLLMEngine are working correctly.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import io
|
||||
import time
|
||||
from statistics import mean, median
|
||||
|
||||
import pytest
|
||||
import soundfile
|
||||
import torch
|
||||
from datasets import Audio, load_dataset
|
||||
from jiwer import wer
|
||||
from transformers.models.whisper.english_normalizer import EnglishTextNormalizer
|
||||
|
||||
from vllm.benchmarks.datasets.datasets import ASRDataset
|
||||
from vllm.multimodal.audio import get_audio_duration
|
||||
from vllm.tokenizers import get_tokenizer
|
||||
|
||||
from ....models.registry import HF_EXAMPLE_MODELS
|
||||
from ....utils import RemoteOpenAIServer
|
||||
|
||||
# Tuned to prevent OOM on 18GB GPUs in transcription correctness tests.
|
||||
MAX_SEQS_FOR_TRANSCRIPTION_TEST = 8
|
||||
GPU_UTIL_FOR_TRANSCRIPTION_TEST = 0.5
|
||||
|
||||
|
||||
def to_bytes(y, sr):
|
||||
buffer = io.BytesIO()
|
||||
soundfile.write(buffer, y, sr, format="WAV")
|
||||
buffer.seek(0)
|
||||
return buffer
|
||||
|
||||
|
||||
def load_audio_sample(audio):
|
||||
# Avoid torchcodec in CI by decoding dataset audio with soundfile.
|
||||
if "array" in audio and "sampling_rate" in audio:
|
||||
return audio["array"], audio["sampling_rate"]
|
||||
|
||||
if audio.get("path"):
|
||||
return soundfile.read(audio["path"], dtype="float32")
|
||||
|
||||
if audio.get("bytes") is not None:
|
||||
return soundfile.read(io.BytesIO(audio["bytes"]), dtype="float32")
|
||||
|
||||
raise ValueError("Audio sample did not contain array, path, or bytes data")
|
||||
|
||||
|
||||
# not all models have a normalizer so use the one from whisper as a standard option
|
||||
normalizer_model_info = HF_EXAMPLE_MODELS.find_hf_info("openai/whisper-large-v3")
|
||||
normalizer_tokenizer = get_tokenizer(
|
||||
"openai/whisper-large-v3",
|
||||
tokenizer_mode=normalizer_model_info.tokenizer_mode,
|
||||
trust_remote_code=normalizer_model_info.trust_remote_code,
|
||||
)
|
||||
normalizer = EnglishTextNormalizer(normalizer_tokenizer.english_spelling_normalizer)
|
||||
|
||||
|
||||
async def transcribe_audio(client, tokenizer, y, sr, extra_body=None):
|
||||
# Send loaded audio directly instead of loading from disk,
|
||||
# don't account for that time though
|
||||
with to_bytes(y, sr) as f:
|
||||
start_time = time.perf_counter()
|
||||
transcription = await client.audio.transcriptions.create(
|
||||
file=f,
|
||||
model=tokenizer.name_or_path,
|
||||
language="en",
|
||||
temperature=0.0,
|
||||
extra_body=extra_body,
|
||||
)
|
||||
end_time = time.perf_counter()
|
||||
# NOTE there's no streaming in transcriptions, can't measure ttft
|
||||
latency = end_time - start_time
|
||||
num_output_tokens = len(
|
||||
tokenizer(transcription.text, add_special_tokens=False).input_ids
|
||||
)
|
||||
return latency, num_output_tokens, transcription.text
|
||||
|
||||
|
||||
async def bound_transcribe(
|
||||
sem, client, tokenizer, audio, sr, reference, extra_body=None
|
||||
):
|
||||
# Use semaphore to limit concurrent requests.
|
||||
async with sem:
|
||||
result = await transcribe_audio(
|
||||
client, tokenizer, audio, sr, extra_body=extra_body
|
||||
)
|
||||
# Normalize *english* output/reference for evaluation.
|
||||
out = normalizer(result[2])
|
||||
ref = normalizer(reference)
|
||||
return result[:2] + (out, ref)
|
||||
|
||||
|
||||
async def process_dataset(model, client, data, concurrent_request, extra_body=None):
|
||||
sem = asyncio.Semaphore(concurrent_request)
|
||||
|
||||
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
|
||||
tokenizer = get_tokenizer(
|
||||
model,
|
||||
tokenizer_mode=model_info.tokenizer_mode,
|
||||
trust_remote_code=model_info.trust_remote_code,
|
||||
)
|
||||
|
||||
# Warmup call as the first `load_audio` server-side is quite slow.
|
||||
audio, sr = load_audio_sample(data[0]["audio"])
|
||||
_ = await bound_transcribe(sem, client, tokenizer, audio, sr, "", extra_body)
|
||||
|
||||
tasks: list[asyncio.Task] = []
|
||||
for sample in data:
|
||||
audio, sr = load_audio_sample(sample["audio"])
|
||||
task = asyncio.create_task(
|
||||
bound_transcribe(
|
||||
sem, client, tokenizer, audio, sr, sample["text"], extra_body
|
||||
)
|
||||
)
|
||||
tasks.append(task)
|
||||
return await asyncio.gather(*tasks)
|
||||
|
||||
|
||||
def print_performance_metrics(results, total_time):
|
||||
latencies = [res[0] for res in results]
|
||||
total_tokens = sum([res[1] for res in results])
|
||||
|
||||
total = len(results)
|
||||
print(f"Total Requests: {total}")
|
||||
print(f"Successful Requests: {len(latencies)}")
|
||||
print(f"Average Latency: {mean(latencies):.4f} seconds")
|
||||
print(f"Median Latency: {median(latencies):.4f} seconds")
|
||||
perc = sorted(latencies)[int(len(latencies) * 0.95) - 1]
|
||||
print(f"95th Percentile Latency: {perc:.4f} seconds")
|
||||
# Throughput
|
||||
req_throughput = len(latencies) / total_time
|
||||
print(f"Estimated req_Throughput: {req_throughput:.2f} requests/s")
|
||||
throughput = total_tokens / total_time
|
||||
print(f"Estimated Throughput: {throughput:.2f} tok/s")
|
||||
|
||||
|
||||
def add_duration(sample):
|
||||
y, sr = load_audio_sample(sample["audio"])
|
||||
sample["duration_ms"] = get_audio_duration(y=y, sr=sr) * 1000
|
||||
return sample
|
||||
|
||||
|
||||
def load_asr_dataset_rows(dataset_repo: str, split="validation", **hf_kwargs):
|
||||
if dataset_repo in ASRDataset.SUPPORTED_DATASET_PATHS:
|
||||
asr_dataset_kwargs = {
|
||||
"dataset_path": dataset_repo,
|
||||
"dataset_split": split,
|
||||
"disable_shuffle": True,
|
||||
"no_stream": True,
|
||||
}
|
||||
for key in ("dataset_subset", "hf_name", "trust_remote_code"):
|
||||
if key in hf_kwargs:
|
||||
asr_dataset_kwargs[key] = hf_kwargs[key]
|
||||
return ASRDataset(**asr_dataset_kwargs).data
|
||||
|
||||
return load_dataset(dataset_repo, split=split, **hf_kwargs)
|
||||
|
||||
|
||||
def load_shortform_eval_dataset(dataset_repo: str, split="validation", **hf_kwargs):
|
||||
## Load and filter the dataset.
|
||||
dataset = load_asr_dataset_rows(dataset_repo, split=split, **hf_kwargs)
|
||||
dataset = dataset.cast_column("audio", Audio(decode=False))
|
||||
if "duration_ms" not in dataset.column_names:
|
||||
# Compute duration to filter.
|
||||
dataset = dataset.map(add_duration)
|
||||
|
||||
# Whisper max supported duration.
|
||||
dataset = dataset.filter(lambda example: example["duration_ms"] < 30000)
|
||||
return dataset
|
||||
|
||||
|
||||
def run_evaluation(
|
||||
model: str,
|
||||
client,
|
||||
dataset,
|
||||
max_concurrent_reqs: int,
|
||||
n_examples: int = -1,
|
||||
print_metrics: bool = True,
|
||||
extra_body=None,
|
||||
):
|
||||
if n_examples > 0:
|
||||
dataset = dataset.select(range(n_examples))
|
||||
start = time.perf_counter()
|
||||
results = asyncio.run(
|
||||
process_dataset(
|
||||
model, client, dataset, max_concurrent_reqs, extra_body=extra_body
|
||||
)
|
||||
)
|
||||
end = time.perf_counter()
|
||||
total_time = end - start
|
||||
print(f"Total Test Time: {total_time:.4f} seconds")
|
||||
if print_metrics:
|
||||
print_performance_metrics(results, total_time)
|
||||
# Compute WER
|
||||
predictions = [res[2] for res in results]
|
||||
references = [res[3] for res in results]
|
||||
wer_score = 100 * wer(references, predictions)
|
||||
print("WER:", wer_score)
|
||||
return wer_score
|
||||
|
||||
|
||||
LONGFORM_DATASET_REPO = ASRDataset.EARNINGS22_CLEANED_DATASET
|
||||
LONGFORM_DATASET_SPLIT = "test"
|
||||
LONGFORM_NUM_SAMPLES = 6
|
||||
|
||||
|
||||
def load_longform_dataset():
|
||||
dataset = load_asr_dataset_rows(
|
||||
LONGFORM_DATASET_REPO,
|
||||
split=LONGFORM_DATASET_SPLIT,
|
||||
)
|
||||
assert len(dataset) >= LONGFORM_NUM_SAMPLES
|
||||
return dataset.select(range(LONGFORM_NUM_SAMPLES))
|
||||
|
||||
|
||||
async def transcribe_audio_path(client, tokenizer, audio_path: str, extra_body=None):
|
||||
with open(audio_path, "rb") as f:
|
||||
start_time = time.perf_counter()
|
||||
transcription = await client.audio.transcriptions.create(
|
||||
file=f,
|
||||
model=tokenizer.name_or_path,
|
||||
language="en",
|
||||
temperature=0.0,
|
||||
extra_body=extra_body,
|
||||
)
|
||||
end_time = time.perf_counter()
|
||||
|
||||
latency = end_time - start_time
|
||||
num_output_tokens = len(
|
||||
tokenizer(transcription.text, add_special_tokens=False).input_ids
|
||||
)
|
||||
return latency, num_output_tokens, transcription.text
|
||||
|
||||
|
||||
async def bound_transcribe_path(
|
||||
sem, client, tokenizer, audio_path, reference, extra_body=None
|
||||
):
|
||||
async with sem:
|
||||
result = await transcribe_audio_path(
|
||||
client, tokenizer, audio_path, extra_body=extra_body
|
||||
)
|
||||
out = normalizer(result[2])
|
||||
ref = normalizer(reference)
|
||||
return result[:2] + (out, ref)
|
||||
|
||||
|
||||
async def process_longform_dataset(
|
||||
model, client, data, concurrent_request, extra_body=None
|
||||
):
|
||||
sem = asyncio.Semaphore(concurrent_request)
|
||||
|
||||
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
|
||||
tokenizer = get_tokenizer(
|
||||
model,
|
||||
tokenizer_mode=model_info.tokenizer_mode,
|
||||
trust_remote_code=model_info.trust_remote_code,
|
||||
)
|
||||
|
||||
warmup_path = data[0]["audio"]["path"]
|
||||
_ = await bound_transcribe_path(sem, client, tokenizer, warmup_path, "", extra_body)
|
||||
|
||||
tasks: list[asyncio.Task] = []
|
||||
for sample in data:
|
||||
audio_path = sample["audio"]["path"]
|
||||
task = asyncio.create_task(
|
||||
bound_transcribe_path(
|
||||
sem, client, tokenizer, audio_path, sample["text"], extra_body
|
||||
)
|
||||
)
|
||||
tasks.append(task)
|
||||
return await asyncio.gather(*tasks)
|
||||
|
||||
|
||||
def run_longform_evaluation(
|
||||
model: str,
|
||||
client,
|
||||
dataset,
|
||||
max_concurrent_reqs: int,
|
||||
print_metrics: bool = True,
|
||||
extra_body=None,
|
||||
):
|
||||
start = time.perf_counter()
|
||||
results = asyncio.run(
|
||||
process_longform_dataset(
|
||||
model, client, dataset, max_concurrent_reqs, extra_body=extra_body
|
||||
)
|
||||
)
|
||||
end = time.perf_counter()
|
||||
total_time = end - start
|
||||
print(f"Total Test Time: {total_time:.4f} seconds")
|
||||
if print_metrics:
|
||||
print_performance_metrics(results, total_time)
|
||||
|
||||
predictions = [res[2] for res in results]
|
||||
references = [res[3] for res in results]
|
||||
wer_score = 100 * wer(references, predictions)
|
||||
print("WER:", wer_score)
|
||||
return wer_score
|
||||
|
||||
|
||||
# alternatives "openai/whisper-large-v2", "openai/whisper-large-v3-turbo"..
|
||||
# NOTE: Expected WER measured with equivalent hf.transformers args:
|
||||
# whisper-large-v3 + esb-datasets-earnings22-validation-tiny-filtered.
|
||||
@pytest.mark.parametrize(
|
||||
"model_config",
|
||||
[
|
||||
("openai/whisper-large-v3", 12.744980),
|
||||
# CohereASR is used to test the variable encoder length code paths
|
||||
("CohereLabs/cohere-transcribe-03-2026", 11.92),
|
||||
],
|
||||
)
|
||||
# Original dataset is 20GB+ in size, hence we use a pre-filtered slice.
|
||||
@pytest.mark.parametrize(
|
||||
"dataset_repo", ["D4nt3/esb-datasets-earnings22-validation-tiny-filtered"]
|
||||
)
|
||||
def test_wer_correctness(
|
||||
model_config, dataset_repo, n_examples=-1, max_concurrent_request=None
|
||||
):
|
||||
model_name, expected_wer = model_config
|
||||
model_info = HF_EXAMPLE_MODELS.find_hf_info(model_name)
|
||||
server_args = [
|
||||
"--enforce-eager",
|
||||
f"--tokenizer_mode={model_info.tokenizer_mode}",
|
||||
f"--max_num_seqs={MAX_SEQS_FOR_TRANSCRIPTION_TEST}",
|
||||
f"--gpu_memory_utilization={GPU_UTIL_FOR_TRANSCRIPTION_TEST}",
|
||||
]
|
||||
if model_info.trust_remote_code:
|
||||
server_args.append("--trust-remote-code")
|
||||
with RemoteOpenAIServer(
|
||||
model_name,
|
||||
server_args,
|
||||
) as remote_server:
|
||||
dataset = load_shortform_eval_dataset(dataset_repo)
|
||||
|
||||
if not max_concurrent_request:
|
||||
# No max concurrency
|
||||
max_concurrent_request = n_examples if n_examples > 0 else len(dataset)
|
||||
|
||||
client = remote_server.get_async_client()
|
||||
wer = run_evaluation(
|
||||
model_name,
|
||||
client,
|
||||
dataset,
|
||||
max_concurrent_request,
|
||||
n_examples,
|
||||
)
|
||||
|
||||
print(f"Expected WER: {expected_wer}, Actual WER: {wer}")
|
||||
|
||||
if expected_wer:
|
||||
wer_atol, wer_rtol = 1e-1, 1e-2
|
||||
max_wer = expected_wer + wer_atol + wer_rtol * abs(expected_wer)
|
||||
assert wer <= max_wer, (
|
||||
f"WER {wer:.6f} exceeds maximum allowed {max_wer:.6f} "
|
||||
f"(baseline {expected_wer:.6f})"
|
||||
)
|
||||
|
||||
|
||||
# 14-22mins of 6 audio samples of total ~115 mins and just 37MB.
|
||||
# checks for long audio transcription correctness and RMS split.
|
||||
@pytest.mark.parametrize(
|
||||
"model_config",
|
||||
[("openai/whisper-large-v3", 9.5)],
|
||||
)
|
||||
def test_long_audio_wer_correctness(model_config):
|
||||
model_name, expected_wer = model_config
|
||||
model_info = HF_EXAMPLE_MODELS.find_hf_info(model_name)
|
||||
server_args = [
|
||||
f"--tokenizer_mode={model_info.tokenizer_mode}",
|
||||
]
|
||||
|
||||
if model_info.trust_remote_code:
|
||||
server_args.append("--trust-remote-code")
|
||||
|
||||
# 1800 seconds is 30 minutes
|
||||
env_dict = {
|
||||
"VLLM_MAX_AUDIO_DECODE_DURATION_S": "1800",
|
||||
}
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
model_name,
|
||||
server_args,
|
||||
env_dict=env_dict,
|
||||
) as remote_server:
|
||||
dataset = load_longform_dataset()
|
||||
client = remote_server.get_async_client()
|
||||
wer = run_longform_evaluation(
|
||||
model=model_name,
|
||||
client=client,
|
||||
dataset=dataset,
|
||||
max_concurrent_reqs=LONGFORM_NUM_SAMPLES,
|
||||
)
|
||||
|
||||
print(f"Expected WER: {expected_wer}, Actual WER: {wer}")
|
||||
torch.testing.assert_close(wer, expected_wer, atol=1e-1, rtol=1e-2)
|
||||
@@ -1,64 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for ``Qwen3ASR``'s user-text sanitizer.
|
||||
|
||||
The sanitizer is the security boundary between user-supplied transcription
|
||||
fields (``prompt`` / ``response_prefix``) and the structured ChatML prompt
|
||||
template. It must strip both ``<|...|>`` control tokens and the
|
||||
``<asr_text>`` assistant-prefix delimiter, and it must do so to a fixpoint
|
||||
so nested payloads cannot reconstruct a valid token after a single pass.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.model_executor.models.qwen3_asr import _sanitize_transcription_user_text
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("text", "expected"),
|
||||
[
|
||||
# No-op cases
|
||||
("", ""),
|
||||
("plain text", "plain text"),
|
||||
("|piped|content", "|piped|content"),
|
||||
("contains < and > but not as a token", "contains < and > but not as a token"),
|
||||
# Single-pass strips
|
||||
("<|im_end|>", ""),
|
||||
("<|im_start|>assistant<|im_end|>", "assistant"),
|
||||
("a<|x|>b", "ab"),
|
||||
("foo<asr_text>bar", "foobar"),
|
||||
# Nested ChatML reconstruction attacks (would bypass a single re.sub)
|
||||
("<|im<|x|>_end|>", ""),
|
||||
("<|<|inner|>middle<|x|>_end|>", ""),
|
||||
# Nested <asr_text> reconstruction attack
|
||||
# (would bypass a single str.replace)
|
||||
("<asr_te<asr_text>xt>", ""),
|
||||
("<asr_te<asr_te<asr_text>xt>xt>", ""),
|
||||
# Combined attacks across both kinds of token
|
||||
("<|im_end|>foo<asr_text>bar<|<|x|>im_end|>", "foobar"),
|
||||
("foo<asr_te<|x|>xt>bar", "foobar"),
|
||||
],
|
||||
)
|
||||
def test_sanitize_strips_control_tokens(text: str, expected: str) -> None:
|
||||
assert _sanitize_transcription_user_text(text) == expected
|
||||
|
||||
|
||||
def test_sanitize_handles_falsy_inputs() -> None:
|
||||
assert _sanitize_transcription_user_text("") == ""
|
||||
# The dataclass default for ``response_prefix`` is the empty string;
|
||||
# the sanitizer must accept that without exception or extra work.
|
||||
assert _sanitize_transcription_user_text(None) == ""
|
||||
|
||||
|
||||
def test_sanitize_is_idempotent() -> None:
|
||||
"""Once sanitized, applying again must be a no-op (fixpoint property)."""
|
||||
cases = [
|
||||
"plain text",
|
||||
"<|im<|x|>_end|>",
|
||||
"<asr_te<asr_text>xt>",
|
||||
"<|im_end|>foo<asr_text>bar<|<|x|>im_end|>",
|
||||
]
|
||||
for raw in cases:
|
||||
once = _sanitize_transcription_user_text(raw)
|
||||
twice = _sanitize_transcription_user_text(once)
|
||||
assert once == twice, f"not idempotent for {raw!r}"
|
||||
-390
@@ -1,390 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""ASR inter-chunk spacing: ``asr_inter_chunk_separator`` and transcription
|
||||
serving (mocked).
|
||||
|
||||
Unit tests cover the helper and ``SupportsTranscription.no_space_languages``.
|
||||
Integration-style tests exercise ``OpenAIServingTranscription`` streaming and
|
||||
``create_transcription`` without loading a model.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import AsyncGenerator
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.config.speech_to_text import SpeechToTextConfig
|
||||
from vllm.entrypoints.generate.base.protocol import RequestResponseMetadata
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.speech_to_text.base.serving import (
|
||||
SpeechToTextBaseServing,
|
||||
asr_inter_chunk_separator,
|
||||
)
|
||||
from vllm.entrypoints.speech_to_text.transcription.protocol import TranscriptionRequest
|
||||
from vllm.entrypoints.speech_to_text.transcription.serving import (
|
||||
OpenAIServingTranscription,
|
||||
)
|
||||
from vllm.model_executor.models.interfaces import (
|
||||
StreamingTranscriptionPostProcessor,
|
||||
SupportsTranscription,
|
||||
)
|
||||
from vllm.model_executor.models.qwen3_asr import Qwen3ASRForConditionalGeneration
|
||||
from vllm.outputs import CompletionOutput, RequestOutput
|
||||
|
||||
# --- Unit: helper + protocol -------------------------------------------------
|
||||
|
||||
|
||||
def test_default_no_space_languages_includes_zh_and_ja():
|
||||
assert SupportsTranscription.no_space_languages == {"ja", "zh"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("language", "expected_sep"),
|
||||
[
|
||||
("en", " "),
|
||||
("EN", " "),
|
||||
("zh", ""),
|
||||
("ZH", ""),
|
||||
("ja", ""),
|
||||
(None, " "),
|
||||
],
|
||||
)
|
||||
def test_asr_inter_chunk_separator_matches_protocol(language, expected_sep):
|
||||
sep = asr_inter_chunk_separator(language, SupportsTranscription.no_space_languages)
|
||||
assert sep == expected_sep
|
||||
|
||||
|
||||
def test_qwen3_asr_stream_processor_passes_plain_text_without_prefix():
|
||||
post_processor = (
|
||||
Qwen3ASRForConditionalGeneration.get_streaming_post_processor_cls()()
|
||||
)
|
||||
|
||||
assert post_processor.process_delta("Hello", False) == "Hello"
|
||||
assert post_processor.process_delta(" world", True) == " world"
|
||||
|
||||
|
||||
def test_qwen3_asr_stream_processor_buffers_prefix_with_leading_space():
|
||||
post_processor = (
|
||||
Qwen3ASRForConditionalGeneration.get_streaming_post_processor_cls()()
|
||||
)
|
||||
|
||||
assert post_processor.process_delta(" language Eng", False) == ""
|
||||
assert post_processor.process_delta("lish<asr", False) == ""
|
||||
assert post_processor.process_delta("_text>Hello", True) == "Hello"
|
||||
|
||||
|
||||
def test_qwen3_asr_stream_processor_keeps_independent_state():
|
||||
processor_cls = Qwen3ASRForConditionalGeneration.get_streaming_post_processor_cls()
|
||||
first_processor = processor_cls()
|
||||
second_processor = processor_cls()
|
||||
|
||||
assert first_processor.process_delta(" language Eng", False) == ""
|
||||
assert second_processor.process_delta("plain text", True) == "plain text"
|
||||
assert first_processor.process_delta("lish<asr_text>Hello", True) == "Hello"
|
||||
|
||||
|
||||
def test_qwen3_asr_stream_processor_emits_finished_incomplete_prefix():
|
||||
post_processor = (
|
||||
Qwen3ASRForConditionalGeneration.get_streaming_post_processor_cls()()
|
||||
)
|
||||
|
||||
assert (
|
||||
post_processor.process_delta(" language English", True) == " language English"
|
||||
)
|
||||
|
||||
|
||||
def test_qwen3_asr_stream_processor_stops_buffering_long_plain_prefix():
|
||||
post_processor = (
|
||||
Qwen3ASRForConditionalGeneration.get_streaming_post_processor_cls()()
|
||||
)
|
||||
text = " language " + ("x" * 50)
|
||||
|
||||
assert post_processor.process_delta(text, False) == text
|
||||
|
||||
|
||||
def test_qwen3_asr_stream_processor_stops_buffering_prefix_with_newline():
|
||||
post_processor = (
|
||||
Qwen3ASRForConditionalGeneration.get_streaming_post_processor_cls()()
|
||||
)
|
||||
text = " language English\nhello"
|
||||
|
||||
assert post_processor.process_delta(text, False) == text
|
||||
|
||||
|
||||
def test_joined_chunks_english_has_space_between():
|
||||
sep = asr_inter_chunk_separator("en", SupportsTranscription.no_space_languages)
|
||||
assert sep.join(["hello", "world"]) == "hello world"
|
||||
|
||||
|
||||
def test_joined_chunks_chinese_has_no_space_between():
|
||||
sep = asr_inter_chunk_separator("zh", SupportsTranscription.no_space_languages)
|
||||
assert sep.join(["你好", "世界"]) == "你好世界"
|
||||
|
||||
|
||||
# --- Integration: serving (no model) -----------------------------------------
|
||||
|
||||
|
||||
class _StubTranscriptionModel:
|
||||
"""Minimal stand-in for a SupportsTranscription implementation (no torch)."""
|
||||
|
||||
no_space_languages: set[str] = {"ja", "zh"}
|
||||
supports_segment_timestamp = False
|
||||
|
||||
@classmethod
|
||||
def get_speech_to_text_config(
|
||||
cls, model_config: ModelConfig, task_type: str
|
||||
) -> SpeechToTextConfig:
|
||||
return SpeechToTextConfig(
|
||||
sample_rate=16000.0,
|
||||
max_audio_clip_s=5.0,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def post_process_output(cls, text: str) -> str:
|
||||
return text
|
||||
|
||||
@classmethod
|
||||
def get_streaming_post_processor_cls(
|
||||
cls,
|
||||
) -> type[StreamingTranscriptionPostProcessor]:
|
||||
return StreamingTranscriptionPostProcessor
|
||||
|
||||
|
||||
def _request_output(text: str, finish_reason: str | None = "stop") -> RequestOutput:
|
||||
return RequestOutput(
|
||||
request_id="rid",
|
||||
prompt=None,
|
||||
prompt_token_ids=None,
|
||||
prompt_logprobs=None,
|
||||
outputs=[
|
||||
CompletionOutput(
|
||||
index=0,
|
||||
text=text,
|
||||
token_ids=(1, 2, 3),
|
||||
cumulative_logprob=None,
|
||||
logprobs=None,
|
||||
finish_reason=finish_reason,
|
||||
)
|
||||
],
|
||||
finished=True,
|
||||
)
|
||||
|
||||
|
||||
def _sse_delta_contents(sse_body: str) -> list[str]:
|
||||
"""Extract ``choices[0].delta.content`` from each ``data:`` line (streaming API)."""
|
||||
contents: list[str] = []
|
||||
for line in sse_body.splitlines():
|
||||
if not line.startswith("data: "):
|
||||
continue
|
||||
payload = line.removeprefix("data: ").strip()
|
||||
if payload == "[DONE]":
|
||||
continue
|
||||
obj = json.loads(payload)
|
||||
for choice in obj.get("choices") or []:
|
||||
delta = choice.get("delta") or {}
|
||||
if "content" in delta:
|
||||
contents.append(delta["content"])
|
||||
return contents
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_transcription_stream_generator_english_inserts_space_between_chunks():
|
||||
"""Online streaming: first output per audio chunk is prefixed with *separator*."""
|
||||
|
||||
async def gen_hello() -> AsyncGenerator[RequestOutput, None]:
|
||||
yield _request_output("hello")
|
||||
|
||||
async def gen_world() -> AsyncGenerator[RequestOutput, None]:
|
||||
yield _request_output("world")
|
||||
|
||||
serving = OpenAIServingTranscription.__new__(OpenAIServingTranscription)
|
||||
serving.enable_force_include_usage = False
|
||||
serving.model_cls = _StubTranscriptionModel
|
||||
serving.streaming_post_processor_cls = (
|
||||
_StubTranscriptionModel.get_streaming_post_processor_cls()
|
||||
)
|
||||
serving.task_type = "transcribe"
|
||||
request = SimpleNamespace(
|
||||
model="stub-model",
|
||||
stream_include_usage=False,
|
||||
stream_continuous_usage_stats=False,
|
||||
)
|
||||
sep = asr_inter_chunk_separator("en", _StubTranscriptionModel.no_space_languages)
|
||||
assert sep == " "
|
||||
|
||||
out_lines: list[str] = []
|
||||
agen = OpenAIServingTranscription.transcription_stream_generator(
|
||||
serving,
|
||||
request=request,
|
||||
result_generator=[gen_hello(), gen_world()],
|
||||
request_id="test-req",
|
||||
request_metadata=RequestResponseMetadata(request_id="test-req"),
|
||||
audio_duration_s=1.0,
|
||||
separator=sep,
|
||||
)
|
||||
async for line in agen:
|
||||
out_lines.append(line)
|
||||
sse = "".join(out_lines)
|
||||
combined = "".join(_sse_delta_contents(sse))
|
||||
assert combined.strip() == "hello world"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_transcription_stream_generator_chinese_no_space_between_chunks():
|
||||
async def gen_a() -> AsyncGenerator[RequestOutput, None]:
|
||||
yield _request_output("你好")
|
||||
|
||||
async def gen_b() -> AsyncGenerator[RequestOutput, None]:
|
||||
yield _request_output("世界")
|
||||
|
||||
serving = OpenAIServingTranscription.__new__(OpenAIServingTranscription)
|
||||
serving.enable_force_include_usage = False
|
||||
serving.model_cls = _StubTranscriptionModel
|
||||
serving.streaming_post_processor_cls = (
|
||||
_StubTranscriptionModel.get_streaming_post_processor_cls()
|
||||
)
|
||||
serving.task_type = "transcribe"
|
||||
request = SimpleNamespace(
|
||||
model="stub-model",
|
||||
stream_include_usage=False,
|
||||
stream_continuous_usage_stats=False,
|
||||
)
|
||||
sep = asr_inter_chunk_separator("zh", _StubTranscriptionModel.no_space_languages)
|
||||
assert sep == ""
|
||||
|
||||
out_lines: list[str] = []
|
||||
agen = OpenAIServingTranscription.transcription_stream_generator(
|
||||
serving,
|
||||
request=request,
|
||||
result_generator=[gen_a(), gen_b()],
|
||||
request_id="test-req-zh",
|
||||
request_metadata=RequestResponseMetadata(request_id="test-req-zh"),
|
||||
audio_duration_s=1.0,
|
||||
separator=sep,
|
||||
)
|
||||
async for line in agen:
|
||||
out_lines.append(line)
|
||||
combined = "".join(_sse_delta_contents("".join(out_lines)))
|
||||
assert combined == "你好世界"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_transcription_stream_generator_strips_qwen3_asr_prefix_per_chunk():
|
||||
async def gen_hello() -> AsyncGenerator[RequestOutput, None]:
|
||||
yield _request_output("language Eng", finish_reason=None)
|
||||
yield _request_output("lish<asr", finish_reason=None)
|
||||
yield _request_output("_text>Hello", finish_reason=None)
|
||||
yield _request_output("")
|
||||
|
||||
async def gen_world() -> AsyncGenerator[RequestOutput, None]:
|
||||
yield _request_output(" language Eng", finish_reason=None)
|
||||
yield _request_output("lish<asr_text>world")
|
||||
|
||||
serving = OpenAIServingTranscription.__new__(OpenAIServingTranscription)
|
||||
serving.enable_force_include_usage = False
|
||||
serving.model_cls = Qwen3ASRForConditionalGeneration
|
||||
serving.streaming_post_processor_cls = (
|
||||
Qwen3ASRForConditionalGeneration.get_streaming_post_processor_cls()
|
||||
)
|
||||
serving.task_type = "transcribe"
|
||||
request = SimpleNamespace(
|
||||
model="stub-qwen3-asr",
|
||||
stream_include_usage=False,
|
||||
stream_continuous_usage_stats=False,
|
||||
)
|
||||
|
||||
out_lines: list[str] = []
|
||||
agen = OpenAIServingTranscription.transcription_stream_generator(
|
||||
serving,
|
||||
request=request,
|
||||
result_generator=[gen_hello(), gen_world()],
|
||||
request_id="test-qwen3-asr",
|
||||
request_metadata=RequestResponseMetadata(request_id="test-qwen3-asr"),
|
||||
audio_duration_s=1.0,
|
||||
separator=" ",
|
||||
)
|
||||
async for line in agen:
|
||||
out_lines.append(line)
|
||||
|
||||
combined = "".join(_sse_delta_contents("".join(out_lines)))
|
||||
assert combined == "Hello world"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_transcription_non_streaming_joins_chunks_by_language():
|
||||
"""``create_transcription`` uses the same separator logic as the helper."""
|
||||
|
||||
async def gen_hello() -> AsyncGenerator[RequestOutput, None]:
|
||||
yield _request_output("hello")
|
||||
|
||||
async def gen_world() -> AsyncGenerator[RequestOutput, None]:
|
||||
yield _request_output("world")
|
||||
|
||||
engine_client = MagicMock()
|
||||
engine_client.model_config = MagicMock()
|
||||
engine_client.model_config.get_diff_sampling_param.return_value = {
|
||||
"max_tokens": 256,
|
||||
"temperature": 0.0,
|
||||
}
|
||||
engine_client.model_config.max_model_len = 8192
|
||||
engine_client.errored = False
|
||||
engine_client.generate.side_effect = [gen_hello(), gen_world()]
|
||||
|
||||
models = MagicMock(spec=OpenAIServingModels)
|
||||
models.lora_requests = {}
|
||||
models.is_base_model.return_value = True
|
||||
|
||||
preprocess_mock = AsyncMock(
|
||||
return_value=([MagicMock(), MagicMock()], 1.0, [0.0, 29.5])
|
||||
)
|
||||
|
||||
with (
|
||||
patch(
|
||||
"vllm.model_executor.model_loader.get_model_cls",
|
||||
return_value=_StubTranscriptionModel,
|
||||
),
|
||||
patch.object(
|
||||
SpeechToTextBaseServing, "_preprocess_speech_to_text", preprocess_mock
|
||||
),
|
||||
):
|
||||
serving = OpenAIServingTranscription(engine_client, models, request_logger=None)
|
||||
|
||||
req_en = TranscriptionRequest.model_construct(
|
||||
file=MagicMock(),
|
||||
model="stub-model",
|
||||
language="en",
|
||||
stream=False,
|
||||
response_format="json",
|
||||
)
|
||||
out_en = await serving.create_transcription(
|
||||
b"\x00\x00", req_en, raw_request=None
|
||||
)
|
||||
assert not isinstance(out_en, ErrorResponse)
|
||||
assert out_en.text == "hello world"
|
||||
|
||||
async def gen_nihao() -> AsyncGenerator[RequestOutput, None]:
|
||||
yield _request_output("你好")
|
||||
|
||||
async def gen_shijie() -> AsyncGenerator[RequestOutput, None]:
|
||||
yield _request_output("世界")
|
||||
|
||||
engine_client.generate.side_effect = [gen_nihao(), gen_shijie()]
|
||||
|
||||
req_zh = TranscriptionRequest.model_construct(
|
||||
file=MagicMock(),
|
||||
model="stub-model",
|
||||
language="zh",
|
||||
stream=False,
|
||||
response_format="json",
|
||||
)
|
||||
out_zh = await serving.create_transcription(
|
||||
b"\x00\x00", req_zh, raw_request=None
|
||||
)
|
||||
assert not isinstance(out_zh, ErrorResponse)
|
||||
assert out_zh.text == "你好世界"
|
||||
@@ -1,130 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Non-object JSON bodies must fail validation cleanly (4xx), not AttributeError (500).
|
||||
|
||||
mode=before validators that call data.get(...) without an isinstance(data, dict)
|
||||
guard raise AttributeError for string/list/scalar bodies and surface as HTTP 500.
|
||||
|
||||
This extends the chat completion coverage added in #51654 to the remaining
|
||||
request models whose before-validators were missing the same guard.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from vllm.entrypoints.generate.generative_scoring.serving import (
|
||||
GenerativeScoringRequest,
|
||||
)
|
||||
from vllm.entrypoints.generate.structured_decisions.protocol import (
|
||||
StructuredDecisionRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.entrypoints.pooling.classify.protocol import ClassificationChatRequest
|
||||
from vllm.entrypoints.pooling.embed.protocol import EmbeddingChatRequest
|
||||
from vllm.entrypoints.pooling.pooling.protocol import PoolingChatRequest
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateRequest
|
||||
from vllm.entrypoints.serve.tokenize.protocol import TokenizeChatRequest
|
||||
from vllm.entrypoints.speech_to_text.transcription.protocol import TranscriptionRequest
|
||||
from vllm.entrypoints.speech_to_text.translation.protocol import TranslationRequest
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
|
||||
REQUEST_MODELS = [
|
||||
GenerativeScoringRequest,
|
||||
CompletionRequest,
|
||||
ResponsesRequest,
|
||||
EmbeddingChatRequest,
|
||||
ClassificationChatRequest,
|
||||
PoolingChatRequest,
|
||||
TokenizeChatRequest,
|
||||
TranscriptionRequest,
|
||||
TranslationRequest,
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("request_model", REQUEST_MODELS, ids=lambda m: m.__name__)
|
||||
@pytest.mark.parametrize(
|
||||
"payload",
|
||||
[
|
||||
"this is not valid json{{{",
|
||||
["not", "an", "object"],
|
||||
42,
|
||||
None,
|
||||
True,
|
||||
],
|
||||
)
|
||||
def test_request_models_reject_non_object_body(request_model, payload):
|
||||
with pytest.raises(ValidationError):
|
||||
request_model.model_validate(payload)
|
||||
|
||||
|
||||
def test_completion_request_still_validates_dict_bodies():
|
||||
"""The guard must not swallow real field-level errors on object bodies."""
|
||||
with pytest.raises(VLLMValidationError, match="prompt"):
|
||||
CompletionRequest.model_validate({"model": "qwen", "prompt": ""})
|
||||
|
||||
|
||||
def test_tokenize_chat_request_still_validates_dict_bodies():
|
||||
with pytest.raises(VLLMValidationError, match="add_generation_prompt"):
|
||||
TokenizeChatRequest.model_validate(
|
||||
{
|
||||
"model": "qwen",
|
||||
"messages": [{"role": "user", "content": "hello"}],
|
||||
"continue_final_message": True,
|
||||
"add_generation_prompt": True,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
CACHE_SALT_MODELS = [
|
||||
(
|
||||
GenerativeScoringRequest,
|
||||
{
|
||||
"query": "Is this city the capital of France?",
|
||||
"items": ["Paris"],
|
||||
"label_token_ids": [9454, 2753],
|
||||
},
|
||||
),
|
||||
(
|
||||
StructuredDecisionRequest,
|
||||
{
|
||||
"state": "hello",
|
||||
"questions": {
|
||||
"lang": {
|
||||
"type": "choice",
|
||||
"criteria": {"English": "Text written in English."},
|
||||
}
|
||||
},
|
||||
},
|
||||
),
|
||||
(CompletionRequest, {"prompt": "hello"}),
|
||||
(ChatCompletionRequest, {"messages": [{"role": "user", "content": "hello"}]}),
|
||||
(ResponsesRequest, {"input": "hello"}),
|
||||
(GenerateRequest, {"token_ids": [1], "sampling_params": {}}),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("request_model", "payload"),
|
||||
CACHE_SALT_MODELS,
|
||||
ids=[m.__name__ for m, _ in CACHE_SALT_MODELS],
|
||||
)
|
||||
def test_request_models_reject_forbidden_cache_salt(request_model, payload):
|
||||
with pytest.raises(VLLMValidationError, match="cache_salt"):
|
||||
request_model.model_validate({**payload, "cache_salt": "/"})
|
||||
|
||||
|
||||
def test_request_models_reject_overlong_cache_salt():
|
||||
with pytest.raises(VLLMValidationError, match="cache_salt"):
|
||||
CompletionRequest.model_validate({"prompt": "hello", "cache_salt": "a" * 129})
|
||||
|
||||
|
||||
def test_request_models_accept_safe_cache_salt():
|
||||
request = CompletionRequest.model_validate(
|
||||
{"prompt": "hello", "cache_salt": "safe-cache_salt-123"}
|
||||
)
|
||||
assert request.cache_salt == "safe-cache_salt-123"
|
||||
@@ -1,471 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Correctness tests for the AMX-only CPU MLA backend: the vendored
|
||||
decode/extend/bmm kernels, the KV cache write, and the ``AMXMLAImpl``
|
||||
backend built on top of them.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_cpu():
|
||||
pytest.skip("skipping CPU-only tests", allow_module_level=True)
|
||||
|
||||
if not torch.cpu._is_amx_tile_supported():
|
||||
pytest.skip("AMX MLA requires an AMX-capable host", allow_module_level=True)
|
||||
|
||||
torch.cpu._init_amx()
|
||||
|
||||
from vllm import _custom_ops as ops # noqa: E402
|
||||
from vllm.utils.torch_utils import set_random_seed # noqa: E402
|
||||
from vllm.v1.attention.backends.mla.amx_mla import ( # noqa: E402
|
||||
AMXMLAImpl,
|
||||
_compute_num_kv_splits,
|
||||
_expand_block_table,
|
||||
)
|
||||
|
||||
KV_LORA_RANK = 64
|
||||
QK_ROPE_HEAD_DIM = 32
|
||||
HEAD_SIZE = KV_LORA_RANK + QK_ROPE_HEAD_DIM # 96, kv-cache row width
|
||||
# The extend/decode kernels transpose by 512-bit lanes: both the cache row
|
||||
# width and the (kv_lora_rank-wide) value width must be 32-element aligned.
|
||||
DTYPE = torch.bfloat16
|
||||
ATOL = 2e-2
|
||||
RTOL = 2e-2
|
||||
# Backend-level tests chain absorb -> attend -> de-absorb through bf16,
|
||||
# accumulating more rounding error than a single kernel op.
|
||||
_IMPL_ATOL = 1.5e-1
|
||||
_IMPL_RTOL = 1.5e-1
|
||||
|
||||
|
||||
def _flatten_cache(kv_cache: torch.Tensor) -> torch.Tensor:
|
||||
"""(num_blocks, block_size, head_size) -> (num_slots, head_size)."""
|
||||
return kv_cache.view(-1, kv_cache.size(-1))
|
||||
|
||||
|
||||
def _random_paged_cache(num_slots: int, head_size: int) -> torch.Tensor:
|
||||
return torch.randn(num_slots, head_size, dtype=DTYPE)
|
||||
|
||||
|
||||
def _ref_latent_attn(
|
||||
q: torch.Tensor,
|
||||
keys: torch.Tensor,
|
||||
scale: float,
|
||||
kv_lora_rank: int,
|
||||
) -> torch.Tensor:
|
||||
"""Plain-PyTorch causal-free latent-space MQA attention for one request.
|
||||
|
||||
q: (num_heads, head_size); keys: (ctx_len, head_size). Values are the
|
||||
first ``kv_lora_rank`` columns of the same latent rows (MLA's K/V
|
||||
aliasing), matching what decode_attention_cpu/extend_attention_cpu
|
||||
compute internally for the MLA-shaped case.
|
||||
"""
|
||||
values = keys[:, :kv_lora_rank]
|
||||
scores = (q.float() @ keys.float().T) * scale
|
||||
probs = torch.softmax(scores, dim=-1)
|
||||
return (probs @ values.float()).to(q.dtype)
|
||||
|
||||
|
||||
def test_bmm_cpu_matches_torch_bmm():
|
||||
set_random_seed(0)
|
||||
# bmm_cpu requires the output's last dim (mat2's out-features) to be a
|
||||
# multiple of 32 (tinygemm's tile width); the contraction dim is free.
|
||||
n, b, p, l = 8, 32, 32, 32 # noqa: E741
|
||||
mat1 = torch.randn(n, b, p, dtype=DTYPE)
|
||||
mat2 = torch.randn(n, l, p, dtype=DTYPE) # (N, OUT, IN) Linear convention
|
||||
|
||||
ref = torch.bmm(mat1.float(), mat2.float().transpose(1, 2)).to(DTYPE)
|
||||
|
||||
out = torch.empty(n, b, l, dtype=DTYPE)
|
||||
ops.bmm_cpu(out, mat1, mat2, False, None)
|
||||
torch.testing.assert_close(out, ref, atol=ATOL, rtol=RTOL)
|
||||
|
||||
packed = torch.ops._C.convert_weight_packed(mat2.contiguous())
|
||||
out_vnni = torch.empty(n, b, l, dtype=DTYPE)
|
||||
ops.bmm_cpu(out_vnni, mat1, packed, True, None)
|
||||
torch.testing.assert_close(out_vnni, ref, atol=ATOL, rtol=RTOL)
|
||||
|
||||
|
||||
def test_concat_and_cache_mla_round_trip():
|
||||
set_random_seed(0)
|
||||
num_tokens = 37
|
||||
num_blocks, block_size = 8, 16
|
||||
kv_cache = torch.zeros(num_blocks, block_size, HEAD_SIZE, dtype=DTYPE)
|
||||
|
||||
kv_c_normed = torch.randn(num_tokens, KV_LORA_RANK, dtype=DTYPE)
|
||||
k_pe = torch.randn(num_tokens, QK_ROPE_HEAD_DIM, dtype=DTYPE)
|
||||
slot_mapping = torch.randperm(num_blocks * block_size)[:num_tokens].to(torch.int64)
|
||||
|
||||
ops.amx_mla_concat_and_cache(kv_c_normed, k_pe, kv_cache, slot_mapping)
|
||||
|
||||
flat = _flatten_cache(kv_cache)
|
||||
read_back = flat[slot_mapping]
|
||||
expected = torch.cat([kv_c_normed, k_pe], dim=-1)
|
||||
torch.testing.assert_close(read_back, expected)
|
||||
|
||||
untouched = torch.ones(num_blocks * block_size, dtype=torch.bool)
|
||||
untouched[slot_mapping] = False
|
||||
assert (flat[untouched] == 0).all()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("seq_lens", [[5], [1, 300, 33], [513, 1, 129, 7]])
|
||||
def test_decode_attention_cpu_matches_reference(seq_lens):
|
||||
set_random_seed(0)
|
||||
num_seqs = len(seq_lens)
|
||||
num_heads = 4
|
||||
block_size = 32
|
||||
max_len = max(seq_lens)
|
||||
max_blocks = (max_len + block_size - 1) // block_size
|
||||
num_blocks = num_seqs * max_blocks
|
||||
|
||||
kv_cache = _random_paged_cache(num_blocks * block_size, HEAD_SIZE)
|
||||
block_table = torch.randperm(num_blocks).view(num_seqs, max_blocks).to(torch.int64)
|
||||
req_to_token = _expand_block_table(block_table, block_size)
|
||||
req_pool_indices = torch.arange(num_seqs, dtype=torch.int64)
|
||||
seq_lens_t = torch.tensor(seq_lens, dtype=torch.int64)
|
||||
scale = HEAD_SIZE**-0.5
|
||||
|
||||
q = torch.randn(num_seqs, num_heads, HEAD_SIZE, dtype=DTYPE)
|
||||
kv_cache_flat = kv_cache.view(-1, 1, HEAD_SIZE)
|
||||
v_buffer = kv_cache_flat[..., :KV_LORA_RANK]
|
||||
|
||||
o = torch.zeros(num_seqs, num_heads, KV_LORA_RANK, dtype=DTYPE)
|
||||
attn_logits = torch.zeros(
|
||||
num_seqs, num_heads, 4, KV_LORA_RANK + 1, dtype=torch.float32
|
||||
)
|
||||
ops.cpu_mla_decode(
|
||||
q,
|
||||
kv_cache_flat,
|
||||
v_buffer,
|
||||
o,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
attn_logits,
|
||||
req_to_token,
|
||||
req_pool_indices,
|
||||
seq_lens_t,
|
||||
scale,
|
||||
0.0,
|
||||
False,
|
||||
0,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
|
||||
flat_cache = _flatten_cache(kv_cache)
|
||||
for i, seq_len in enumerate(seq_lens):
|
||||
token_ids = req_to_token[i, :seq_len]
|
||||
keys = flat_cache[token_ids]
|
||||
ref = _ref_latent_attn(q[i], keys, scale, KV_LORA_RANK)
|
||||
torch.testing.assert_close(o[i], ref, atol=ATOL, rtol=RTOL)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"prefix_lens,extend_lens",
|
||||
[
|
||||
([0], [9]), # fresh prefill, no cached prefix
|
||||
([64], [17]), # continuation of a cached prefix
|
||||
([0, 40, 128], [23, 5, 31]), # mixed batch
|
||||
],
|
||||
)
|
||||
def test_extend_attention_cpu_matches_reference(prefix_lens, extend_lens):
|
||||
set_random_seed(0)
|
||||
num_seqs = len(prefix_lens)
|
||||
num_heads = 4
|
||||
block_size = 32
|
||||
seq_lens = [p + e for p, e in zip(prefix_lens, extend_lens)]
|
||||
max_len = max(seq_lens)
|
||||
max_blocks = (max_len + block_size - 1) // block_size
|
||||
num_blocks = num_seqs * max_blocks
|
||||
|
||||
kv_cache = _random_paged_cache(num_blocks * block_size, HEAD_SIZE)
|
||||
block_table = torch.randperm(num_blocks).view(num_seqs, max_blocks).to(torch.int64)
|
||||
req_to_token = _expand_block_table(block_table, block_size)
|
||||
req_pool_indices = torch.arange(num_seqs, dtype=torch.int64)
|
||||
seq_lens_t = torch.tensor(seq_lens, dtype=torch.int64)
|
||||
extend_seq_lens_t = torch.tensor(extend_lens, dtype=torch.int64)
|
||||
extend_start_loc = torch.cumsum(
|
||||
torch.tensor([0, *extend_lens[:-1]], dtype=torch.int64), dim=0
|
||||
)
|
||||
scale = HEAD_SIZE**-0.5
|
||||
total_new_tokens = sum(extend_lens)
|
||||
|
||||
# The new tokens' latent K/V, already written into the cache (mirrors
|
||||
# do_kv_cache_update running before forward_mha in real usage).
|
||||
k_extend_flat = torch.randn(total_new_tokens, HEAD_SIZE, dtype=DTYPE)
|
||||
flat_cache = _flatten_cache(kv_cache)
|
||||
for i in range(num_seqs):
|
||||
prefix_len = prefix_lens[i]
|
||||
ext_len = extend_lens[i]
|
||||
new_token_ids = req_to_token[i, prefix_len : prefix_len + ext_len]
|
||||
start = extend_start_loc[i].item()
|
||||
flat_cache[new_token_ids] = k_extend_flat[start : start + ext_len]
|
||||
|
||||
q_extend = torch.randn(total_new_tokens, num_heads, HEAD_SIZE, dtype=DTYPE)
|
||||
k_extend = k_extend_flat.unsqueeze(1)
|
||||
v_extend = k_extend[..., :KV_LORA_RANK]
|
||||
kv_cache_flat = kv_cache.view(-1, 1, HEAD_SIZE)
|
||||
v_buffer = kv_cache_flat[..., :KV_LORA_RANK]
|
||||
|
||||
o_extend = torch.empty(total_new_tokens, num_heads, KV_LORA_RANK, dtype=DTYPE)
|
||||
ops.cpu_mla_extend(
|
||||
q_extend,
|
||||
k_extend,
|
||||
v_extend,
|
||||
o_extend,
|
||||
kv_cache_flat,
|
||||
v_buffer,
|
||||
req_to_token,
|
||||
req_pool_indices,
|
||||
seq_lens_t,
|
||||
extend_seq_lens_t,
|
||||
extend_start_loc,
|
||||
max(extend_lens),
|
||||
scale,
|
||||
0.0,
|
||||
False,
|
||||
0,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
|
||||
for i in range(num_seqs):
|
||||
prefix_len = prefix_lens[i]
|
||||
ext_len = extend_lens[i]
|
||||
start = extend_start_loc[i].item()
|
||||
for t in range(ext_len):
|
||||
visible = prefix_len + t + 1
|
||||
token_ids = req_to_token[i, :visible]
|
||||
keys = flat_cache[token_ids]
|
||||
ref = _ref_latent_attn(q_extend[start + t], keys, scale, KV_LORA_RANK)
|
||||
torch.testing.assert_close(o_extend[start + t], ref, atol=ATOL, rtol=RTOL)
|
||||
|
||||
|
||||
class _FakeLinear:
|
||||
"""Minimal duck-typed stand-in for kv_b_proj: only .weight and
|
||||
.quant_method are read by get_and_maybe_dequant_weights()."""
|
||||
|
||||
def __init__(self, weight: torch.Tensor):
|
||||
self.weight = weight
|
||||
self.quant_method = None
|
||||
|
||||
|
||||
def _make_amx_mla_impl(num_heads, qk_nope_head_dim, v_head_dim, kv_lora_rank):
|
||||
weight = torch.randn(
|
||||
num_heads * (qk_nope_head_dim + v_head_dim), kv_lora_rank, dtype=DTYPE
|
||||
)
|
||||
kv_b_proj = _FakeLinear(weight)
|
||||
impl = AMXMLAImpl(
|
||||
num_heads=num_heads,
|
||||
head_size=kv_lora_rank + QK_ROPE_HEAD_DIM,
|
||||
scale=(qk_nope_head_dim + QK_ROPE_HEAD_DIM) ** -0.5,
|
||||
num_kv_heads=1,
|
||||
alibi_slopes=None,
|
||||
sliding_window=None,
|
||||
kv_cache_dtype="auto",
|
||||
logits_soft_cap=None,
|
||||
attn_type="decoder",
|
||||
kv_sharing_target_layer_name=None,
|
||||
q_lora_rank=None,
|
||||
kv_lora_rank=kv_lora_rank,
|
||||
qk_nope_head_dim=qk_nope_head_dim,
|
||||
qk_rope_head_dim=QK_ROPE_HEAD_DIM,
|
||||
qk_head_dim=qk_nope_head_dim + QK_ROPE_HEAD_DIM,
|
||||
v_head_dim=v_head_dim,
|
||||
kv_b_proj=kv_b_proj,
|
||||
)
|
||||
impl.process_weights_after_loading(DTYPE)
|
||||
return impl, weight
|
||||
|
||||
|
||||
class _FakePrefillMetadata:
|
||||
def __init__(
|
||||
self, block_table, cpu_seq_lens, query_start_loc, req_to_token, req_pool_indices
|
||||
):
|
||||
self.block_table = block_table
|
||||
self.cpu_seq_lens = cpu_seq_lens
|
||||
query_start_loc_i64 = query_start_loc.to(torch.int64)
|
||||
extend_seq_lens = query_start_loc_i64[1:] - query_start_loc_i64[:-1]
|
||||
self.extend_seq_lens = extend_seq_lens
|
||||
self.extend_start_loc = query_start_loc_i64[:-1]
|
||||
self.max_len_extend = int(extend_seq_lens.max().item())
|
||||
self.req_to_token = req_to_token
|
||||
self.req_pool_indices = req_pool_indices
|
||||
|
||||
|
||||
class _FakeDecodeMetadata:
|
||||
def __init__(self, block_table, seq_lens, req_to_token, req_pool_indices):
|
||||
self.block_table = block_table
|
||||
self.seq_lens_i64 = seq_lens.to(torch.int64)
|
||||
self.req_to_token = req_to_token
|
||||
self.req_pool_indices = req_pool_indices
|
||||
self.num_kv_splits = _compute_num_kv_splits(
|
||||
int(seq_lens.max().item()), current_platform.num_compute_units()
|
||||
)
|
||||
|
||||
|
||||
class _FakeAttnMetadata:
|
||||
def __init__(self, decode=None, prefill=None, max_seq_len=0):
|
||||
self.decode = decode
|
||||
self.prefill = prefill
|
||||
self.max_seq_len = max_seq_len
|
||||
|
||||
|
||||
def test_amx_mla_impl_forward_mqa_matches_reference(default_vllm_config):
|
||||
"""forward_mqa receives already-absorbed Q (as MLAAttention.forward_impl
|
||||
would produce via layer.W_UK_T) and must reproduce plain per-head MLA
|
||||
decode attention through W_UK/W_UV."""
|
||||
set_random_seed(1)
|
||||
num_heads, qk_nope_head_dim, v_head_dim, kv_lora_rank = 4, 32, 32, 64
|
||||
impl, kv_b_weight = _make_amx_mla_impl(
|
||||
num_heads, qk_nope_head_dim, v_head_dim, kv_lora_rank
|
||||
)
|
||||
w_uk, w_uv = kv_b_weight.T.view(
|
||||
kv_lora_rank, num_heads, qk_nope_head_dim + v_head_dim
|
||||
).split([qk_nope_head_dim, v_head_dim], dim=-1)
|
||||
|
||||
seq_lens = [1, 40, 5]
|
||||
num_seqs = len(seq_lens)
|
||||
block_size = 32
|
||||
max_blocks = (max(seq_lens) + block_size - 1) // block_size
|
||||
num_blocks = num_seqs * max_blocks
|
||||
head_size = kv_lora_rank + QK_ROPE_HEAD_DIM
|
||||
|
||||
kv_cache = torch.randn(num_blocks, block_size, head_size, dtype=DTYPE)
|
||||
block_table = torch.randperm(num_blocks).view(num_seqs, max_blocks).to(torch.int64)
|
||||
req_to_token = _expand_block_table(block_table, block_size)
|
||||
|
||||
q_nope = torch.randn(num_seqs, num_heads, qk_nope_head_dim, dtype=DTYPE)
|
||||
q_pe = torch.randn(num_seqs, num_heads, QK_ROPE_HEAD_DIM, dtype=DTYPE)
|
||||
# Pre-absorb, mirroring MLAAttention.forward_impl's own bmm against W_UK_T.
|
||||
ql_nope = torch.einsum("bnp,lnp->bnl", q_nope.float(), w_uk.float()).to(DTYPE)
|
||||
|
||||
attn_metadata = _FakeAttnMetadata(
|
||||
decode=_FakeDecodeMetadata(
|
||||
block_table=block_table,
|
||||
seq_lens=torch.tensor(seq_lens, dtype=torch.int64),
|
||||
req_to_token=req_to_token,
|
||||
req_pool_indices=torch.arange(num_seqs, dtype=torch.int64),
|
||||
),
|
||||
max_seq_len=max(seq_lens),
|
||||
)
|
||||
o, lse = impl.forward_mqa((ql_nope, q_pe), kv_cache, attn_metadata, layer=None)
|
||||
assert lse is None
|
||||
# De-absorb, mirroring MLAAttention.forward_impl's own bmm against
|
||||
# layer._v_up_proj, to get back into real per-head V space for
|
||||
# comparison against a from-scratch reference.
|
||||
out_real = torch.einsum("bnl,lnv->bnv", o.float(), w_uv.float())
|
||||
|
||||
flat_cache = _flatten_cache(kv_cache)
|
||||
scale = impl.scale
|
||||
for i, seq_len in enumerate(seq_lens):
|
||||
token_ids = req_to_token[i, :seq_len]
|
||||
keys_latent = flat_cache[token_ids] # (L, head_size)
|
||||
k_nope_latent, k_pe = keys_latent.split(
|
||||
[kv_lora_rank, QK_ROPE_HEAD_DIM], dim=-1
|
||||
)
|
||||
k_real = torch.einsum("lp,pnd->nld", k_nope_latent.float(), w_uk.float())
|
||||
k_pe_b = k_pe.float().unsqueeze(0).expand(num_heads, -1, -1)
|
||||
k_full = torch.cat([k_real, k_pe_b], dim=-1) # (num_heads, L, qk_head_dim)
|
||||
q_full = torch.cat(
|
||||
[q_nope[i], q_pe[i]], dim=-1
|
||||
).float() # (num_heads, qk_head_dim)
|
||||
scores = torch.einsum("nd,nld->nl", q_full, k_full) * scale
|
||||
probs = torch.softmax(scores, dim=-1)
|
||||
v_real = torch.einsum("lp,pnv->nlv", k_nope_latent.float(), w_uv.float())
|
||||
ref = torch.einsum("nl,nlv->nv", probs, v_real)
|
||||
torch.testing.assert_close(out_real[i], ref, atol=_IMPL_ATOL, rtol=_IMPL_RTOL)
|
||||
|
||||
|
||||
def test_amx_mla_impl_forward_mha_matches_reference(default_vllm_config):
|
||||
"""forward_mha receives raw unabsorbed Q and must attend correctly for a
|
||||
prefill batch with a mix of fresh and continued (cached-prefix)
|
||||
sequences."""
|
||||
set_random_seed(2)
|
||||
num_heads, qk_nope_head_dim, v_head_dim, kv_lora_rank = 4, 32, 32, 64
|
||||
impl, kv_b_weight = _make_amx_mla_impl(
|
||||
num_heads, qk_nope_head_dim, v_head_dim, kv_lora_rank
|
||||
)
|
||||
w_uk, w_uv = kv_b_weight.T.view(
|
||||
kv_lora_rank, num_heads, qk_nope_head_dim + v_head_dim
|
||||
).split([qk_nope_head_dim, v_head_dim], dim=-1)
|
||||
|
||||
prefix_lens = [0, 20]
|
||||
extend_lens = [13, 7]
|
||||
seq_lens = [p + e for p, e in zip(prefix_lens, extend_lens)]
|
||||
num_seqs = len(prefix_lens)
|
||||
block_size = 32
|
||||
max_blocks = (max(seq_lens) + block_size - 1) // block_size
|
||||
num_blocks = num_seqs * max_blocks
|
||||
head_size = kv_lora_rank + QK_ROPE_HEAD_DIM
|
||||
|
||||
kv_cache = torch.randn(num_blocks, block_size, head_size, dtype=DTYPE)
|
||||
block_table = torch.randperm(num_blocks).view(num_seqs, max_blocks).to(torch.int64)
|
||||
req_to_token = _expand_block_table(block_table, block_size)
|
||||
flat_cache = _flatten_cache(kv_cache)
|
||||
|
||||
total_new_tokens = sum(extend_lens)
|
||||
query_start_loc = torch.tensor(
|
||||
[0, *torch.cumsum(torch.tensor(extend_lens), dim=0).tolist()],
|
||||
dtype=torch.int64,
|
||||
)
|
||||
|
||||
q_nope = torch.randn(total_new_tokens, num_heads, qk_nope_head_dim, dtype=DTYPE)
|
||||
q_pe = torch.randn(total_new_tokens, num_heads, QK_ROPE_HEAD_DIM, dtype=DTYPE)
|
||||
q = torch.cat([q_nope, q_pe], dim=-1)
|
||||
|
||||
kv_c_normed = torch.randn(total_new_tokens, kv_lora_rank, dtype=DTYPE)
|
||||
k_pe = torch.randn(total_new_tokens, 1, QK_ROPE_HEAD_DIM, dtype=DTYPE)
|
||||
|
||||
# Write the new tokens' latent K/V into the cache ahead of time, mirroring
|
||||
# do_kv_cache_update running before forward_mha in real usage.
|
||||
for i in range(num_seqs):
|
||||
prefix_len = prefix_lens[i]
|
||||
ext_len = extend_lens[i]
|
||||
new_token_ids = req_to_token[i, prefix_len : prefix_len + ext_len]
|
||||
start = query_start_loc[i].item()
|
||||
flat_cache[new_token_ids, :kv_lora_rank] = kv_c_normed[start : start + ext_len]
|
||||
flat_cache[new_token_ids, kv_lora_rank:] = k_pe[start : start + ext_len, 0]
|
||||
|
||||
attn_metadata = _FakeAttnMetadata(
|
||||
prefill=_FakePrefillMetadata(
|
||||
block_table=block_table,
|
||||
cpu_seq_lens=torch.tensor(seq_lens, dtype=torch.int64),
|
||||
query_start_loc=query_start_loc,
|
||||
req_to_token=req_to_token,
|
||||
req_pool_indices=torch.arange(num_seqs, dtype=torch.int64),
|
||||
)
|
||||
)
|
||||
output = torch.empty(total_new_tokens, num_heads * v_head_dim, dtype=DTYPE)
|
||||
impl.forward_mha(
|
||||
q, kv_c_normed, k_pe, kv_cache, attn_metadata, k_scale=None, output=output
|
||||
)
|
||||
out_view = output.view(total_new_tokens, num_heads, v_head_dim).float()
|
||||
|
||||
scale = impl.scale
|
||||
for i in range(num_seqs):
|
||||
prefix_len = prefix_lens[i]
|
||||
ext_len = extend_lens[i]
|
||||
start = query_start_loc[i].item()
|
||||
for t in range(ext_len):
|
||||
visible = prefix_len + t + 1
|
||||
token_ids = req_to_token[i, :visible]
|
||||
keys_latent = flat_cache[token_ids]
|
||||
k_nope_latent, k_pe_ref = keys_latent.split(
|
||||
[kv_lora_rank, QK_ROPE_HEAD_DIM], dim=-1
|
||||
)
|
||||
k_real = torch.einsum("lp,pnd->nld", k_nope_latent.float(), w_uk.float())
|
||||
k_pe_b = k_pe_ref.float().unsqueeze(0).expand(num_heads, -1, -1)
|
||||
k_full = torch.cat([k_real, k_pe_b], dim=-1)
|
||||
q_full = torch.cat([q_nope[start + t], q_pe[start + t]], dim=-1).float()
|
||||
scores = torch.einsum("nd,nld->nl", q_full, k_full) * scale
|
||||
probs = torch.softmax(scores, dim=-1)
|
||||
v_real = torch.einsum("lp,pnv->nlv", k_nope_latent.float(), w_uv.float())
|
||||
ref = torch.einsum("nl,nlv->nv", probs, v_real)
|
||||
torch.testing.assert_close(
|
||||
out_view[start + t], ref, atol=_IMPL_ATOL, rtol=_IMPL_RTOL
|
||||
)
|
||||
@@ -1,919 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from contextlib import contextmanager
|
||||
from typing import Literal
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import (
|
||||
AttentionConfig,
|
||||
CacheConfig,
|
||||
ParallelConfig,
|
||||
VllmConfig,
|
||||
set_current_vllm_config,
|
||||
)
|
||||
from vllm.config.cache import CacheDType
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.platforms.cpu import CpuPlatform
|
||||
from vllm.platforms.interface import DeviceCapability
|
||||
|
||||
if current_platform.is_cuda():
|
||||
from vllm.platforms.cuda import CudaPlatform
|
||||
else:
|
||||
CudaPlatform = None
|
||||
|
||||
if current_platform.is_rocm():
|
||||
from vllm.platforms.rocm import RocmPlatform
|
||||
else:
|
||||
RocmPlatform = None # type: ignore[misc] # Unavailable platform import.
|
||||
|
||||
from vllm.v1.attention.backend import AttentionType
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.attention.selector import (
|
||||
AttentionSelectorConfig,
|
||||
_cached_get_attn_backend,
|
||||
get_attn_backend,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def clear_cache():
|
||||
"""Clear lru cache to ensure each test case runs without caching."""
|
||||
_cached_get_attn_backend.cache_clear()
|
||||
|
||||
|
||||
# Define MLA and non-MLA backends separately
|
||||
DEVICE_MLA_BACKENDS = {
|
||||
"cuda": [
|
||||
"TRITON_MLA",
|
||||
"FLASHMLA",
|
||||
"FLASHINFER_MLA",
|
||||
"FLASH_ATTN_MLA",
|
||||
"CUTLASS_MLA",
|
||||
],
|
||||
"hip": ["TRITON_MLA", "ROCM_AITER_MLA"],
|
||||
"cpu": [],
|
||||
}
|
||||
|
||||
DEVICE_REGULAR_ATTN_BACKENDS = {
|
||||
"cuda": ["FLASHINFER", "FLASH_ATTN"],
|
||||
"hip": ["ROCM_ATTN"],
|
||||
"cpu": ["CPU_ATTN"],
|
||||
}
|
||||
|
||||
DEVICE_MLA_BLOCK_SIZES = {
|
||||
"cuda": [16, 64], # CUDA supports both standard and extended block sizes
|
||||
"hip": [16, 1], # HIP requires special handling for block_size=1
|
||||
# "cpu": [16] # CPU uses fixed block size from test cases
|
||||
"cpu": [], # FIXME(woosuk): Temporarily disable CPU tests
|
||||
}
|
||||
|
||||
|
||||
def generate_params():
|
||||
is_rocm = current_platform.is_rocm()
|
||||
params = []
|
||||
device_list = ["cuda", "cpu"] if not is_rocm else ["hip", "cpu"]
|
||||
for use_mla in [True, False]:
|
||||
for device in device_list:
|
||||
backends = (
|
||||
DEVICE_MLA_BACKENDS[device]
|
||||
if use_mla
|
||||
else DEVICE_REGULAR_ATTN_BACKENDS[device]
|
||||
)
|
||||
for name in backends:
|
||||
block_sizes = (
|
||||
[128]
|
||||
if name == "CUTLASS_MLA"
|
||||
else DEVICE_MLA_BLOCK_SIZES[device]
|
||||
if use_mla
|
||||
else [16]
|
||||
)
|
||||
for block_size in block_sizes:
|
||||
for use_dcp in [False, True] if device != "cpu" else [False]:
|
||||
params.append(
|
||||
pytest.param(
|
||||
device,
|
||||
name,
|
||||
use_mla,
|
||||
block_size,
|
||||
use_dcp,
|
||||
id=(
|
||||
f"{device}_{name}_mla_{str(use_mla)[0]}"
|
||||
f"_blks{block_size}_dcp{use_dcp}"
|
||||
),
|
||||
)
|
||||
)
|
||||
return params
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"device, name, use_mla, block_size, use_dcp", generate_params()
|
||||
)
|
||||
def test_backend_selection(
|
||||
device: str,
|
||||
name: str,
|
||||
use_mla: bool,
|
||||
block_size: int,
|
||||
use_dcp: bool,
|
||||
):
|
||||
"""Supported GPU backends remain selectable when DCP is enabled."""
|
||||
# Create AttentionConfig with the specified backend
|
||||
attention_config = AttentionConfig(backend=AttentionBackendEnum[name])
|
||||
cache_config = CacheConfig(block_size=block_size)
|
||||
vllm_config = VllmConfig(
|
||||
attention_config=attention_config,
|
||||
cache_config=cache_config,
|
||||
parallel_config=ParallelConfig(
|
||||
# Skip GPU-count autodetection: selection does not launch workers.
|
||||
distributed_executor_backend="mp" if use_dcp else None,
|
||||
tensor_parallel_size=2 if use_dcp else 1,
|
||||
decode_context_parallel_size=2 if use_dcp else 1,
|
||||
),
|
||||
)
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
if device == "cpu":
|
||||
with patch("vllm.platforms.current_platform", CpuPlatform()):
|
||||
backend = get_attn_backend(16, torch.float16, None)
|
||||
assert backend.get_name() == "CPU_ATTN"
|
||||
|
||||
elif device == "hip":
|
||||
if RocmPlatform is None:
|
||||
pytest.skip("RocmPlatform not available")
|
||||
with patch("vllm.platforms.current_platform", RocmPlatform()):
|
||||
if use_mla:
|
||||
# ROCm MLA backend logic:
|
||||
# - TRITON_MLA: supported when block_size != 1
|
||||
# - ROCM_AITER_MLA: supported when block_size == 1
|
||||
# If backend is forced but doesn't match block_size,
|
||||
# should raise ValueError
|
||||
|
||||
if name == "TRITON_MLA" and block_size == 1:
|
||||
# TRITON_MLA doesn't support block_size == 1
|
||||
with pytest.raises(ValueError):
|
||||
get_attn_backend(576, torch.float16, None, use_mla=use_mla)
|
||||
else:
|
||||
# Valid backend-block_size combination
|
||||
backend = get_attn_backend(
|
||||
576, torch.float16, None, use_mla=use_mla
|
||||
)
|
||||
expected = name
|
||||
assert backend.get_name() == expected
|
||||
else:
|
||||
if use_dcp:
|
||||
with pytest.raises(ValueError, match="DCP not supported"):
|
||||
get_attn_backend(32, torch.float16, None, use_mla=use_mla)
|
||||
else:
|
||||
backend = get_attn_backend(
|
||||
32, torch.float16, None, use_mla=use_mla
|
||||
)
|
||||
expected = "ROCM_ATTN"
|
||||
assert backend.get_name() == expected
|
||||
|
||||
elif device == "cuda":
|
||||
if CudaPlatform is None:
|
||||
pytest.skip("CudaPlatform not available")
|
||||
with patch("vllm.platforms.current_platform", CudaPlatform()):
|
||||
capability = torch.cuda.get_device_capability()
|
||||
if use_mla:
|
||||
# CUDA MLA backend logic:
|
||||
# - CUTLASS_MLA: only supported with block_size == 128
|
||||
# and Blackwell GPUs (SM 10.x), V1 only
|
||||
# - FLASHINFER_MLA: only supported on Blackwell GPUs
|
||||
# (SM 10.x), V1 only
|
||||
# - FLASHMLA: only supported with block_size == 64
|
||||
# - FLASH_ATTN_MLA: V1 only
|
||||
# - TRITON_MLA: fallback for other cases
|
||||
|
||||
if name == "CUTLASS_MLA":
|
||||
if block_size != 128:
|
||||
# CUTLASS_MLA only supports block_size == 128
|
||||
pytest.skip("CUTLASS_MLA only supports block_size 128")
|
||||
if capability[0] != 10:
|
||||
pytest.skip("CUTLASS MLA is not supported on this platform")
|
||||
backend = get_attn_backend(
|
||||
576, torch.float16, None, use_mla=use_mla
|
||||
)
|
||||
expected = "CUTLASS_MLA"
|
||||
assert backend.get_name() == expected
|
||||
elif name == "FLASHINFER_MLA":
|
||||
if capability[0] != 10:
|
||||
pytest.skip(
|
||||
"FlashInfer MLA is not supported on this platform"
|
||||
)
|
||||
if block_size not in [32, 64]:
|
||||
# FlashInfer MLA only supports block_size 32 or 64
|
||||
pytest.skip(
|
||||
"FlashInfer MLA only supports block_size 32 or 64"
|
||||
)
|
||||
backend = get_attn_backend(
|
||||
576, torch.float16, None, use_mla=use_mla
|
||||
)
|
||||
expected = "FLASHINFER_MLA"
|
||||
assert backend.get_name() == expected
|
||||
elif name == "FLASHMLA":
|
||||
if block_size != 64:
|
||||
# FlashMLA only supports block_size == 64
|
||||
pytest.skip("FlashMLA only supports block_size 64")
|
||||
from vllm.v1.attention.backends.mla.flashmla import (
|
||||
is_flashmla_dense_supported,
|
||||
)
|
||||
|
||||
is_supported, _ = is_flashmla_dense_supported()
|
||||
if not is_supported:
|
||||
pytest.skip("FlashMLA not supported on this platform")
|
||||
backend = get_attn_backend(
|
||||
576,
|
||||
torch.float16,
|
||||
None,
|
||||
use_mla=use_mla,
|
||||
)
|
||||
expected = name
|
||||
assert backend.get_name() == expected
|
||||
elif name == "FLASH_ATTN_MLA":
|
||||
from vllm.v1.attention.backends.fa_utils import (
|
||||
flash_attn_supports_mla,
|
||||
)
|
||||
|
||||
if not flash_attn_supports_mla():
|
||||
pytest.skip(
|
||||
"FlashAttention MLA not supported on this platform"
|
||||
)
|
||||
backend = get_attn_backend(
|
||||
576, torch.float16, None, use_mla=use_mla
|
||||
)
|
||||
expected = "FLASH_ATTN_MLA"
|
||||
assert backend.get_name() == expected
|
||||
else:
|
||||
# TRITON_MLA or other fallback
|
||||
backend = get_attn_backend(
|
||||
576, torch.float16, None, use_mla=use_mla
|
||||
)
|
||||
expected = "TRITON_MLA"
|
||||
assert backend.get_name() == expected
|
||||
elif name == "FLASHINFER":
|
||||
backend = get_attn_backend(64, torch.float16, None, use_mla=use_mla)
|
||||
expected = "FLASHINFER"
|
||||
assert backend.get_name() == expected
|
||||
elif name == "FLASH_ATTN":
|
||||
backend = get_attn_backend(32, torch.float16, None, use_mla=use_mla)
|
||||
expected = "FLASH_ATTN"
|
||||
assert backend.get_name() == expected
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="GPU attention backends"
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"backend_name",
|
||||
[
|
||||
"FLASH_ATTN",
|
||||
"FLASH_ATTN_DIFFKV",
|
||||
"FLASHINFER",
|
||||
"TRITON_MLA",
|
||||
"ROCM_AITER_MLA",
|
||||
"ROCM_AITER_TRITON_MLA",
|
||||
"CUTLASS_MLA",
|
||||
"FLASHMLA",
|
||||
"FLASHINFER_MLA",
|
||||
"TOKENSPEED_MLA",
|
||||
"FLASH_ATTN_MLA",
|
||||
"FLASHMLA_SPARSE",
|
||||
"FLASHINFER_MLA_SPARSE",
|
||||
],
|
||||
)
|
||||
def test_supported_backend_preserves_dcp_eligibility(backend_name, default_vllm_config):
|
||||
"""DCP opt-ins preserve eligibility and unrelated configuration restrictions."""
|
||||
if backend_name.startswith("ROCM_") and not current_platform.is_rocm():
|
||||
pytest.skip("ROCm-specific backend")
|
||||
try:
|
||||
backend_cls = AttentionBackendEnum[backend_name].get_class()
|
||||
except ImportError as error:
|
||||
pytest.skip(f"Optional backend dependency unavailable: {error}")
|
||||
|
||||
config = AttentionSelectorConfig(
|
||||
head_size=576 if backend_cls.is_mla() else 128,
|
||||
dtype=torch.bfloat16,
|
||||
kv_cache_dtype="auto",
|
||||
block_size=backend_cls.get_preferred_block_size(64),
|
||||
use_mla=backend_cls.is_mla(),
|
||||
use_sparse=backend_cls.is_sparse(),
|
||||
)
|
||||
kwargs = dict(
|
||||
device_capability=current_platform.get_device_capability(), **config._asdict()
|
||||
)
|
||||
invalid_without_dcp = backend_cls.validate_configuration(**kwargs)
|
||||
kwargs["use_dcp"] = True
|
||||
assert backend_cls.validate_configuration(**kwargs) == invalid_without_dcp
|
||||
|
||||
|
||||
@pytest.mark.parametrize("device", ["cpu", "cuda", "hip"])
|
||||
def test_fp32_fallback(device: str):
|
||||
"""Test attention backend selection with fp32."""
|
||||
# Use default config (no backend specified)
|
||||
vllm_config = VllmConfig()
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
if device == "cpu":
|
||||
with patch("vllm.platforms.current_platform", CpuPlatform()):
|
||||
backend = get_attn_backend(16, torch.float32, None)
|
||||
assert backend.get_name() == "CPU_ATTN"
|
||||
|
||||
elif device == "cuda":
|
||||
if CudaPlatform is None:
|
||||
pytest.skip("CudaPlatform not available")
|
||||
with patch("vllm.platforms.current_platform", CudaPlatform()):
|
||||
backend = get_attn_backend(16, torch.float32, None)
|
||||
assert backend.get_name() == "FLEX_ATTENTION"
|
||||
|
||||
elif device == "hip":
|
||||
if RocmPlatform is None:
|
||||
pytest.skip("RocmPlatform not available")
|
||||
# ROCm backends do not support head_size=16 (minimum is 32).
|
||||
# No known HuggingFace transformer model uses head_size=16.
|
||||
# Revisit if a real model with this head size is identified
|
||||
# and accuracy-tested.
|
||||
with (
|
||||
patch("vllm.platforms.current_platform", RocmPlatform()),
|
||||
pytest.raises(ValueError, match="No valid attention backend"),
|
||||
):
|
||||
get_attn_backend(16, torch.float32, None)
|
||||
|
||||
|
||||
def test_invalid_backend():
|
||||
"""Test that invalid attention backend names raise ValueError."""
|
||||
with (
|
||||
pytest.raises(ValueError),
|
||||
):
|
||||
# Invalid backend name should raise ValueError when creating enum
|
||||
AttentionConfig(backend=AttentionBackendEnum["INVALID"])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("auto_value", ["auto", "AUTO", "Auto"])
|
||||
def test_auto_backend_string(auto_value: str):
|
||||
"""Test that 'auto' string value triggers automatic backend selection."""
|
||||
# Using "auto" should result in backend=None (automatic selection)
|
||||
attention_config = AttentionConfig(backend=auto_value)
|
||||
assert attention_config.backend is None
|
||||
|
||||
|
||||
def test_auto_backend_selection_behavior():
|
||||
"""Test that 'auto' backend behaves same as None (automatic selection)."""
|
||||
# Create config with explicit "auto"
|
||||
auto_config = AttentionConfig(backend="auto")
|
||||
|
||||
# Create config with None (default)
|
||||
none_config = AttentionConfig(backend=None)
|
||||
|
||||
# Both should have backend=None
|
||||
assert auto_config.backend is None
|
||||
assert none_config.backend is None
|
||||
|
||||
# Both configs should result in the same automatic backend selection
|
||||
vllm_config_auto = VllmConfig(attention_config=auto_config)
|
||||
vllm_config_none = VllmConfig(attention_config=none_config)
|
||||
|
||||
with (
|
||||
set_current_vllm_config(vllm_config_auto),
|
||||
patch("vllm.platforms.current_platform", CpuPlatform()),
|
||||
):
|
||||
backend_auto = get_attn_backend(16, torch.float16, None)
|
||||
|
||||
_cached_get_attn_backend.cache_clear()
|
||||
|
||||
with (
|
||||
set_current_vllm_config(vllm_config_none),
|
||||
patch("vllm.platforms.current_platform", CpuPlatform()),
|
||||
):
|
||||
backend_none = get_attn_backend(16, torch.float16, None)
|
||||
|
||||
# Both should select the same backend
|
||||
assert backend_auto.get_name() == backend_none.get_name()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"backend_name,flash_attn_version,should_succeed",
|
||||
[
|
||||
("FLASH_ATTN", 3, True), # FA3 supports per-head quant scales
|
||||
("FLASH_ATTN", 2, False), # FA2 does not support per-head quant scales
|
||||
("FLASHINFER", None, False), # FlashInfer does not support
|
||||
("FLEX_ATTENTION", None, False), # Flex does not support
|
||||
],
|
||||
)
|
||||
@pytest.mark.skipif(
|
||||
current_platform.is_rocm(),
|
||||
reason="Attention backend FA3 is not supported on ROCm. This test can't succeed.",
|
||||
)
|
||||
def test_per_head_quant_scales_backend_selection(
|
||||
backend_name: str, flash_attn_version: Literal[2, 3, 4] | None, should_succeed: bool
|
||||
):
|
||||
"""Test backend selection when use_per_head_quant_scales=True."""
|
||||
# Clear cache to ensure fresh backend selection
|
||||
_cached_get_attn_backend.cache_clear()
|
||||
|
||||
attention_config = AttentionConfig(
|
||||
backend=AttentionBackendEnum[backend_name],
|
||||
flash_attn_version=flash_attn_version,
|
||||
)
|
||||
cache_config = CacheConfig(block_size=64)
|
||||
vllm_config = VllmConfig(
|
||||
attention_config=attention_config, cache_config=cache_config
|
||||
)
|
||||
|
||||
if CudaPlatform is None:
|
||||
pytest.skip("CudaPlatform not available")
|
||||
with (
|
||||
set_current_vllm_config(vllm_config),
|
||||
patch("vllm.platforms.current_platform", CudaPlatform()),
|
||||
):
|
||||
if backend_name == "FLASH_ATTN" and flash_attn_version == 3:
|
||||
if not torch.cuda.is_available():
|
||||
pytest.skip("FA3 requires CUDA")
|
||||
capability = torch.cuda.get_device_capability()
|
||||
if capability[0] != 9:
|
||||
pytest.skip("FA3 is only supported on Hopper (SM 9.x) GPUs")
|
||||
|
||||
if should_succeed:
|
||||
backend = get_attn_backend(
|
||||
head_size=128,
|
||||
dtype=torch.float16,
|
||||
kv_cache_dtype="fp8",
|
||||
use_per_head_quant_scales=True,
|
||||
)
|
||||
assert backend.get_name() == backend_name
|
||||
else:
|
||||
with pytest.raises(ValueError) as exc_info:
|
||||
get_attn_backend(
|
||||
head_size=128,
|
||||
dtype=torch.float16,
|
||||
kv_cache_dtype="fp8",
|
||||
use_per_head_quant_scales=True,
|
||||
)
|
||||
assert backend_name in str(exc_info.value)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
CudaPlatform is None, reason="CUDA platform is required for this test"
|
||||
)
|
||||
@pytest.mark.parametrize("head_size", [64, 80, 96, 128, 192, 256])
|
||||
def test_int4_per_token_head_head_size_selection(head_size: int):
|
||||
"""INT4 per-token-head KV cache is selectable at any even head size.
|
||||
|
||||
The Hadamard rotation the INT4 write path applies is now defined for
|
||||
non-power-of-two rows as well (it runs block-diagonally, see
|
||||
``fast_hadamard_transform``), so a model such as Phi-3 (head_size=96) must
|
||||
keep selecting TRITON_ATTN instead of being rejected.
|
||||
"""
|
||||
vllm_config = VllmConfig(cache_config=CacheConfig(block_size=64))
|
||||
with (
|
||||
set_current_vllm_config(vllm_config),
|
||||
patch("vllm.platforms.current_platform", CudaPlatform()),
|
||||
):
|
||||
backend = get_attn_backend(
|
||||
head_size=head_size,
|
||||
dtype=torch.float16,
|
||||
kv_cache_dtype="int4_per_token_head",
|
||||
)
|
||||
assert backend.get_name() == "TRITON_ATTN"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"backend_name,use_non_causal,should_succeed",
|
||||
[
|
||||
("FLASH_ATTN", True, True), # FlashAttn supports non-causal
|
||||
("FLASH_ATTN", False, True), # FlashAttn also works with causal
|
||||
]
|
||||
+ (
|
||||
[
|
||||
("FLASHINFER", True, True), # FlashInfer supports non-causal
|
||||
("FLASHINFER", False, True), # FlashInfer works with causal
|
||||
]
|
||||
if CudaPlatform is not None
|
||||
else []
|
||||
),
|
||||
)
|
||||
def test_non_causal_backend_selection(
|
||||
backend_name: str, use_non_causal: bool, should_succeed: bool
|
||||
):
|
||||
"""Test that use_non_causal on AttentionConfig controls backend filtering.
|
||||
|
||||
DFlashProposer sets use_non_causal=True on the draft model's
|
||||
AttentionConfig so only non-causal-capable backends are selected.
|
||||
The target model keeps use_non_causal=False (default) and can use
|
||||
any backend.
|
||||
"""
|
||||
_cached_get_attn_backend.cache_clear()
|
||||
|
||||
attention_config = AttentionConfig(
|
||||
backend=AttentionBackendEnum[backend_name],
|
||||
use_non_causal=use_non_causal,
|
||||
)
|
||||
cache_config = CacheConfig(block_size=16)
|
||||
vllm_config = VllmConfig(
|
||||
attention_config=attention_config, cache_config=cache_config
|
||||
)
|
||||
|
||||
platform = CudaPlatform or RocmPlatform
|
||||
if platform is None:
|
||||
pytest.skip("CudaPlatform and RocmPlatform are not available")
|
||||
with (
|
||||
set_current_vllm_config(vllm_config),
|
||||
patch("vllm.platforms.current_platform", platform()),
|
||||
):
|
||||
if should_succeed:
|
||||
backend = get_attn_backend(
|
||||
head_size=128,
|
||||
dtype=torch.float16,
|
||||
kv_cache_dtype=None,
|
||||
)
|
||||
assert backend.get_name() == backend_name
|
||||
else:
|
||||
with pytest.raises(ValueError) as exc_info:
|
||||
get_attn_backend(
|
||||
head_size=128,
|
||||
dtype=torch.float16,
|
||||
kv_cache_dtype=None,
|
||||
)
|
||||
assert "non-causal" in str(exc_info.value).lower()
|
||||
|
||||
|
||||
def test_non_causal_autoselect_backend():
|
||||
"""Test that when backend=None with use_non_causal=True, auto-selection
|
||||
picks a compatible backend.
|
||||
|
||||
This simulates the DFlash scenario where the user doesn't specify
|
||||
--attention-backend or --speculative-config.attention_backend.
|
||||
The drafter inherits backend=None and auto-selects a backend that
|
||||
supports non-causal attention.
|
||||
"""
|
||||
_cached_get_attn_backend.cache_clear()
|
||||
|
||||
attention_config = AttentionConfig(
|
||||
backend=None,
|
||||
use_non_causal=True,
|
||||
)
|
||||
cache_config = CacheConfig(block_size=16)
|
||||
vllm_config = VllmConfig(
|
||||
attention_config=attention_config, cache_config=cache_config
|
||||
)
|
||||
|
||||
if CudaPlatform is None:
|
||||
pytest.skip("CudaPlatform not available")
|
||||
with (
|
||||
set_current_vllm_config(vllm_config),
|
||||
patch("vllm.platforms.current_platform", CudaPlatform()),
|
||||
):
|
||||
backend = get_attn_backend(
|
||||
head_size=128,
|
||||
dtype=torch.float16,
|
||||
kv_cache_dtype=None,
|
||||
)
|
||||
assert backend.supports_non_causal()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"kv_cache_dtype",
|
||||
[
|
||||
"fp8_e5m2",
|
||||
"fp8_ds_mla",
|
||||
"fp8_inc",
|
||||
"nvfp4",
|
||||
"nvfp4_4over6",
|
||||
"fp8_per_token_head",
|
||||
"int8_per_token_head",
|
||||
],
|
||||
)
|
||||
def test_flash_attn_rejects_unhandled_kv_cache_dtypes(kv_cache_dtype: CacheDType):
|
||||
"""FlashAttentionBackend must not claim support for kv_cache dtypes
|
||||
that it cannot handle."""
|
||||
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
|
||||
|
||||
assert not FlashAttentionBackend.supports_kv_cache_dtype(kv_cache_dtype)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kv_cache_dtype", ["fp8", "fp8_e4m3"])
|
||||
def test_flash_attn_accepts_handled_fp8_variants(
|
||||
kv_cache_dtype: CacheDType, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
"""FlashAttentionBackend must accept the two fp8 dtypes it can actually
|
||||
handle: 'fp8' (alias for fp8_e4m3fn) and 'fp8_e4m3'."""
|
||||
import vllm.v1.attention.backends.fa_utils as fa_utils_mod
|
||||
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
|
||||
|
||||
# The fp8 decision is made in fa_utils, using its own current_platform
|
||||
# binding, so patch is_xpu there (not on flash_attn's) to stay robust to
|
||||
# import order across earlier tests that patch vllm.platforms.current_platform.
|
||||
monkeypatch.setattr(fa_utils_mod.current_platform, "is_xpu", lambda: True)
|
||||
assert FlashAttentionBackend.supports_kv_cache_dtype(kv_cache_dtype)
|
||||
|
||||
|
||||
blackwell_only = pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="FA4 is CUDA-only"
|
||||
)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _blackwell(vllm_config=None):
|
||||
platform = MagicMock()
|
||||
platform.is_xpu.return_value = False
|
||||
platform.is_rocm.return_value = False
|
||||
platform.get_device_capability.return_value = DeviceCapability(10, 0)
|
||||
with (
|
||||
patch("vllm.v1.attention.backends.fa_utils.current_platform", platform),
|
||||
patch(
|
||||
"vllm.vllm_flash_attn.flash_attn_interface.is_fa_version_supported",
|
||||
return_value=True,
|
||||
),
|
||||
patch("vllm.config.get_current_vllm_config_or_none", return_value=vllm_config),
|
||||
patch(
|
||||
"vllm.v1.attention.backends.flash_attn.get_current_vllm_config_or_none",
|
||||
return_value=vllm_config,
|
||||
),
|
||||
):
|
||||
yield
|
||||
|
||||
|
||||
def _hd256_config(
|
||||
*,
|
||||
is_mm_prefix_lm=False,
|
||||
rswa_window=None,
|
||||
dcp_size=1,
|
||||
softcap=None,
|
||||
head_size=256,
|
||||
cache_dtype="auto",
|
||||
):
|
||||
vllm_config = MagicMock()
|
||||
vllm_config.attention_config.flash_attn_version = None
|
||||
vllm_config.model_config.is_mm_prefix_lm = is_mm_prefix_lm
|
||||
vllm_config.model_config.rswa_window = rswa_window
|
||||
vllm_config.model_config.hf_text_config.attn_logit_softcapping = softcap
|
||||
vllm_config.model_config.get_head_size.return_value = head_size
|
||||
vllm_config.cache_config.cache_dtype = cache_dtype
|
||||
vllm_config.parallel_config.decode_context_parallel_size = dcp_size
|
||||
return vllm_config
|
||||
|
||||
|
||||
@blackwell_only
|
||||
@pytest.mark.parametrize(
|
||||
"kwargs,config_kwargs,expected",
|
||||
[
|
||||
({}, {}, 4),
|
||||
({"supports_fa4_hd256": False}, {}, 2),
|
||||
({"has_sinks": True}, {}, 2),
|
||||
({"requires_softcap": True}, {}, 2),
|
||||
# Larger block sizes normalize to 128.
|
||||
({"kv_cache_block_size": 16}, {}, 2),
|
||||
({"kv_cache_block_size": 64}, {}, 2),
|
||||
({"kv_cache_block_size": 256}, {}, 4),
|
||||
({}, {"is_mm_prefix_lm": True}, 2),
|
||||
({}, {"rswa_window": 512}, 2),
|
||||
({}, {"dcp_size": 2}, 2),
|
||||
({}, {"softcap": 50.0}, 2),
|
||||
({}, {"cache_dtype": "fp8"}, 2),
|
||||
({"head_size": 128, "kv_cache_block_size": 16}, {}, 4),
|
||||
({"head_size": 192, "head_size_v": 128, "kv_cache_block_size": 16}, {}, 4),
|
||||
({"head_size": 256, "head_size_v": 128, "kv_cache_block_size": 16}, {}, 2),
|
||||
({"head_size": 256, "head_size_v": 64}, {}, 2),
|
||||
],
|
||||
)
|
||||
def test_fa4_hd256_fallback_matrix(kwargs, config_kwargs, expected):
|
||||
from vllm.v1.attention.backends.fa_utils import get_flash_attn_version
|
||||
|
||||
kwargs = {
|
||||
"head_size": 256,
|
||||
"kv_cache_block_size": 128,
|
||||
"supports_fa4_hd256": True,
|
||||
**kwargs,
|
||||
}
|
||||
with _blackwell(_hd256_config(head_size=kwargs["head_size"], **config_kwargs)):
|
||||
assert get_flash_attn_version(**kwargs) == expected
|
||||
|
||||
|
||||
@blackwell_only
|
||||
@pytest.mark.parametrize(
|
||||
"backend_name,config_kwargs,expected",
|
||||
[
|
||||
("FLASH_ATTN", {}, 128),
|
||||
("FLASH_ATTN", {"softcap": 50.0}, 16),
|
||||
("FLASH_ATTN", {"head_size": 128}, 16),
|
||||
("FLASH_ATTN", {"cache_dtype": "fp8"}, 16),
|
||||
("FLASH_ATTN_DIFFKV", {}, 16),
|
||||
],
|
||||
)
|
||||
def test_fa4_hd256_block_size_advertisement(
|
||||
backend_name: str, config_kwargs: dict, expected: int
|
||||
):
|
||||
from vllm.v1.attention.backend import MultipleOf
|
||||
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
|
||||
from vllm.v1.attention.backends.flash_attn_diffkv import (
|
||||
FlashAttentionDiffKVBackend,
|
||||
)
|
||||
|
||||
backend = {
|
||||
"FLASH_ATTN": FlashAttentionBackend,
|
||||
"FLASH_ATTN_DIFFKV": FlashAttentionDiffKVBackend,
|
||||
}[backend_name]
|
||||
vllm_config = _hd256_config(**config_kwargs)
|
||||
with _blackwell(vllm_config):
|
||||
(size,) = backend.get_supported_kernel_block_sizes()
|
||||
preferred = backend.get_preferred_block_size(16)
|
||||
assert preferred == expected
|
||||
if expected == 128:
|
||||
assert size == 128
|
||||
else:
|
||||
assert isinstance(size, MultipleOf) and size.base == expected
|
||||
|
||||
|
||||
@blackwell_only
|
||||
def test_fa4_hd256_mm_prefix_deselects_flash_attn():
|
||||
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
|
||||
|
||||
with _blackwell(_hd256_config(is_mm_prefix_lm=True)):
|
||||
assert (
|
||||
FlashAttentionBackend.supports_combination(
|
||||
head_size=256,
|
||||
dtype=torch.bfloat16,
|
||||
kv_cache_dtype=None,
|
||||
block_size=128,
|
||||
use_mla=False,
|
||||
has_sink=False,
|
||||
use_sparse=False,
|
||||
use_mm_prefix=True,
|
||||
device_capability=DeviceCapability(10, 0),
|
||||
)
|
||||
is not None
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda()
|
||||
or not current_platform.is_device_capability_family(100),
|
||||
reason="requires a Blackwell GPU",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"attn_type,sliding_window,expected",
|
||||
[
|
||||
(AttentionType.DECODER, None, 4),
|
||||
(AttentionType.DECODER, 512, 4),
|
||||
# Local encoder attention lacks the required seqused tensors.
|
||||
(AttentionType.ENCODER_ONLY, None, 4),
|
||||
(AttentionType.ENCODER_ONLY, 512, 2),
|
||||
],
|
||||
)
|
||||
def test_fa4_hd256_impl_selection(attn_type, sliding_window, expected):
|
||||
from vllm.v1.attention.backends.flash_attn import FlashAttentionImpl
|
||||
|
||||
# Implementations are constructed before the platform updates the block size.
|
||||
with set_current_vllm_config(VllmConfig()):
|
||||
impl = FlashAttentionImpl(
|
||||
num_heads=8,
|
||||
head_size=256,
|
||||
scale=0.0625,
|
||||
num_kv_heads=8,
|
||||
alibi_slopes=None,
|
||||
sliding_window=sliding_window,
|
||||
kv_cache_dtype="auto",
|
||||
attn_type=attn_type,
|
||||
)
|
||||
assert impl.vllm_flash_attn_version == expected
|
||||
assert impl.fa4_hd256 == (expected == 4)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def blackwell_selection():
|
||||
with (
|
||||
patch.object(
|
||||
type(current_platform),
|
||||
"get_device_capability",
|
||||
return_value=DeviceCapability(10, 0),
|
||||
),
|
||||
patch(
|
||||
"vllm.v1.attention.backends.fa_utils.is_fa_version_supported",
|
||||
return_value=True,
|
||||
),
|
||||
):
|
||||
yield
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def hopper_selection():
|
||||
with (
|
||||
patch.object(
|
||||
type(current_platform),
|
||||
"get_device_capability",
|
||||
return_value=DeviceCapability(9, 0),
|
||||
),
|
||||
patch(
|
||||
"vllm.v1.attention.backends.fa_utils.is_fa_version_supported",
|
||||
return_value=True,
|
||||
),
|
||||
):
|
||||
yield
|
||||
|
||||
|
||||
@blackwell_only
|
||||
@pytest.mark.parametrize("use_mm_prefix", [False, True])
|
||||
@pytest.mark.parametrize("flash_attn_version", [None, 3, 4])
|
||||
def test_hopper_mm_prefix_selects_triton_flash_attn(
|
||||
use_mm_prefix, flash_attn_version, hopper_selection
|
||||
):
|
||||
"""Hopper selects the composite when its causal route resolves FA3 or FA4."""
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
|
||||
config = EngineArgs(
|
||||
model="google/gemma-4-31B-it",
|
||||
dtype="bfloat16",
|
||||
attention_config=AttentionConfig(flash_attn_version=flash_attn_version),
|
||||
).create_engine_config()
|
||||
with set_current_vllm_config(config):
|
||||
backend = get_attn_backend(
|
||||
256, torch.bfloat16, None, use_mm_prefix=use_mm_prefix
|
||||
)
|
||||
expected = "TRITON_FLASH_ATTN" if use_mm_prefix else "FLASH_ATTN"
|
||||
assert backend.get_name() == expected
|
||||
|
||||
|
||||
@blackwell_only
|
||||
@pytest.mark.parametrize("use_mm_prefix", [False, True])
|
||||
@pytest.mark.parametrize("kv_cache_dtype", [None, "fp8_e4m3"])
|
||||
def test_mm_prefix_selects_composite_without_changing_causal_default(
|
||||
use_mm_prefix, kv_cache_dtype, blackwell_selection
|
||||
):
|
||||
"""The image-mask requirement must reach CUDA's automatic backend priority."""
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
|
||||
config = EngineArgs(
|
||||
model="google/gemma-4-31B-it", dtype="bfloat16"
|
||||
).create_engine_config()
|
||||
with set_current_vllm_config(config):
|
||||
backend = get_attn_backend(
|
||||
256, torch.bfloat16, kv_cache_dtype, use_mm_prefix=use_mm_prefix
|
||||
)
|
||||
expected = "TRITON_FLASHINFER" if use_mm_prefix else "FLASHINFER"
|
||||
assert backend.get_name() == expected
|
||||
|
||||
|
||||
@blackwell_only
|
||||
@pytest.mark.parametrize(
|
||||
"feature_kwargs,reason",
|
||||
[
|
||||
({"use_rswa": True}, "R-SWA not supported"),
|
||||
({"kv_cache_dtype": "int8_per_token_head"}, "kv_cache_dtype not supported"),
|
||||
({"block_size": 16}, "block_size not supported"),
|
||||
({"use_dcp": True}, "DCP not supported"),
|
||||
({"use_pcp": True}, "PCP not supported"),
|
||||
({"use_adaptive_verification": True}, "device-cpu query lens mismatch"),
|
||||
],
|
||||
)
|
||||
def test_composite_rejects_features_not_shared_by_both_routes(feature_kwargs, reason):
|
||||
"""Automatic selection must not admit features only one child can execute."""
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.v1.attention.backends.triton_flashinfer import TritonFlashInferBackend
|
||||
|
||||
config = EngineArgs(
|
||||
model="google/gemma-4-31B-it", dtype="bfloat16"
|
||||
).create_engine_config()
|
||||
args = dict(
|
||||
head_size=256,
|
||||
dtype=torch.bfloat16,
|
||||
kv_cache_dtype="auto",
|
||||
block_size=128,
|
||||
use_mla=False,
|
||||
has_sink=False,
|
||||
use_sparse=False,
|
||||
use_mm_prefix=True,
|
||||
use_per_head_quant_scales=False,
|
||||
device_capability=DeviceCapability(10, 0),
|
||||
attn_type=AttentionType.DECODER,
|
||||
)
|
||||
args.update(feature_kwargs)
|
||||
with set_current_vllm_config(config):
|
||||
reasons = TritonFlashInferBackend.validate_configuration(**args)
|
||||
assert any(reason in message for message in reasons), reasons
|
||||
|
||||
|
||||
@blackwell_only
|
||||
def test_rswa_selection_does_not_reuse_causal_result(blackwell_selection):
|
||||
"""An R-SWA requirement must invalidate a previously cached causal selection."""
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
|
||||
config = EngineArgs(
|
||||
model="google/gemma-4-31B-it",
|
||||
dtype="bfloat16",
|
||||
attention_config=AttentionConfig(
|
||||
backend=AttentionBackendEnum.TRITON_FLASHINFER
|
||||
),
|
||||
).create_engine_config()
|
||||
with set_current_vllm_config(config):
|
||||
assert (
|
||||
get_attn_backend(256, torch.bfloat16, None).get_name()
|
||||
== "TRITON_FLASHINFER"
|
||||
)
|
||||
config.model_config.model_arch_config.rswa_window = 128
|
||||
with pytest.raises(ValueError, match="R-SWA"):
|
||||
get_attn_backend(256, torch.bfloat16, None)
|
||||
config.attention_config.backend = AttentionBackendEnum.TRITON_ATTN
|
||||
assert get_attn_backend(256, torch.bfloat16, None).get_name() == "TRITON_ATTN"
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,410 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import random
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.sparse_attn_indexer import as_fp32_logits
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import (
|
||||
_ceil_to_ue8m0,
|
||||
calc_diff,
|
||||
fp8_fp4_mqa_logits,
|
||||
fp8_fp4_paged_mqa_logits,
|
||||
get_num_sms,
|
||||
get_paged_mqa_logits_metadata,
|
||||
mqa_logits_mx_inputs,
|
||||
mqa_logits_use_mx_scales,
|
||||
native_next_n_supported,
|
||||
)
|
||||
from vllm.utils.import_utils import has_deep_gemm
|
||||
from vllm.utils.math_utils import cdiv
|
||||
|
||||
|
||||
def encode_indexer_scales(sf: torch.Tensor) -> torch.Tensor:
|
||||
"""Per-token power-of-two fp32 scales as the indexer K cache stores them:
|
||||
four equal packed UE8M0 bytes under MX scales, else the fp32 itself."""
|
||||
if not mqa_logits_use_mx_scales():
|
||||
return sf
|
||||
ue8m0 = (sf.view(torch.int32) >> 23).to(torch.uint8)
|
||||
return ue8m0[..., None].repeat_interleave(4, dim=-1).view(torch.int32)[..., 0]
|
||||
|
||||
|
||||
def kv_cache_cast_to_fp8(x: torch.Tensor) -> torch.Tensor:
|
||||
# x: (num_blocks, block_size, 1, head_dim)
|
||||
num_blocks, block_size, num_heads, head_dim = x.shape
|
||||
assert num_heads == 1
|
||||
x_amax = x.abs().float().amax(dim=3, keepdim=True).clamp(1e-4)
|
||||
sf = _ceil_to_ue8m0(x_amax / 448.0)
|
||||
x_scaled = (x * (1.0 / sf)).to(torch.float8_e4m3fn)
|
||||
x_fp8 = torch.empty(
|
||||
(num_blocks, block_size * (head_dim + 4)),
|
||||
device=x.device,
|
||||
dtype=torch.uint8,
|
||||
)
|
||||
x_fp8[:, : block_size * head_dim] = x_scaled.view(
|
||||
num_blocks, block_size * head_dim
|
||||
).view(dtype=torch.uint8)
|
||||
x_fp8[:, block_size * head_dim :] = encode_indexer_scales(
|
||||
sf.view(num_blocks, block_size)
|
||||
).view(dtype=torch.uint8)
|
||||
return x_fp8.view(num_blocks, block_size, num_heads, head_dim + 4)
|
||||
|
||||
|
||||
def per_custom_dims_cast_to_fp8(
|
||||
x: torch.Tensor, dims: tuple, use_ue8m0: bool
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
excluded_dims = tuple([i for i in range(x.dim()) if i not in set(dims)])
|
||||
x_amax = x.abs().float().amax(dim=excluded_dims, keepdim=True).clamp(1e-4)
|
||||
sf = x_amax / 448.0
|
||||
sf = _ceil_to_ue8m0(sf) if use_ue8m0 else sf
|
||||
x_scaled = (x * (1.0 / sf)).to(torch.float8_e4m3fn)
|
||||
return x_scaled, sf.squeeze()
|
||||
|
||||
|
||||
def _generate_cp_test_data(seq_len: int, seq_len_kv: int):
|
||||
assert seq_len_kv % seq_len == 0 and seq_len % 2 == 0
|
||||
chunk_size = seq_len // 2
|
||||
cp_size = seq_len_kv // seq_len
|
||||
cp_id = cp_size // 3
|
||||
ks = torch.zeros(seq_len, dtype=torch.int, device="cuda")
|
||||
ke = torch.zeros(seq_len, dtype=torch.int, device="cuda")
|
||||
for i in range(chunk_size):
|
||||
ke[i] = cp_id * chunk_size + i
|
||||
ke[i + chunk_size] = (cp_size * 2 - 1 - cp_id) * chunk_size + i
|
||||
return ks, ke
|
||||
|
||||
|
||||
def _ref_fp8_mqa_logits(
|
||||
q: torch.Tensor,
|
||||
kv: torch.Tensor,
|
||||
weights: torch.Tensor,
|
||||
cu_seqlen_ks: torch.Tensor,
|
||||
cu_seqlen_ke: torch.Tensor,
|
||||
):
|
||||
seq_len_kv = kv.shape[0]
|
||||
|
||||
k = kv
|
||||
q = q.float()
|
||||
k = k.float()
|
||||
|
||||
mask_lo = (
|
||||
torch.arange(0, seq_len_kv, device="cuda")[None, :] >= cu_seqlen_ks[:, None]
|
||||
)
|
||||
mask_hi = (
|
||||
torch.arange(0, seq_len_kv, device="cuda")[None, :] < cu_seqlen_ke[:, None]
|
||||
)
|
||||
mask = mask_lo & mask_hi
|
||||
score = torch.einsum("mhd,nd->hmn", q, k)
|
||||
logits = (score.relu() * weights.unsqueeze(-1).transpose(0, 1)).sum(dim=0)
|
||||
logits = logits.masked_fill(~mask, float("-inf"))
|
||||
|
||||
return logits
|
||||
|
||||
|
||||
def _packed_requests_test_data(seq_len: int, seq_len_kv: int):
|
||||
# Two causal prefill requests packed into one KV buffer, so the second
|
||||
# request's rows start at a nonzero K position.
|
||||
half, half_kv = seq_len // 2, seq_len_kv // 2
|
||||
rows = torch.arange(seq_len, dtype=torch.int, device="cuda")
|
||||
ks = torch.where(rows < half, 0, half_kv).to(torch.int)
|
||||
ke = ks + half_kv - half + 1 + rows % half
|
||||
return ks, ke
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
|
||||
@pytest.mark.skipif(not has_deep_gemm(), reason="DeepGEMM not available")
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.has_device_capability(90), reason="SM90 and SM100 only"
|
||||
)
|
||||
@pytest.mark.parametrize("layout", ["cp", "no_cp", "packed"])
|
||||
def test_deepgemm_fp8_mqa_logits(layout: str):
|
||||
"""FP8 indexer prefill logits, compressed to each row's valid K span."""
|
||||
torch.manual_seed(0)
|
||||
random.seed(0)
|
||||
num_heads, head_dim = 32, 128
|
||||
seq_len, seq_len_kv = 512, 1024
|
||||
q = torch.randn(seq_len, num_heads, head_dim, device="cuda", dtype=torch.bfloat16)
|
||||
kv = torch.randn(seq_len_kv, head_dim, device="cuda", dtype=torch.bfloat16)
|
||||
weights = torch.randn(seq_len, num_heads, device="cuda", dtype=torch.float32)
|
||||
|
||||
if layout == "cp":
|
||||
ks, ke = _generate_cp_test_data(seq_len, seq_len_kv)
|
||||
elif layout == "no_cp":
|
||||
ks = torch.zeros(seq_len, dtype=torch.int, device="cuda")
|
||||
ke = torch.arange(seq_len, dtype=torch.int, device="cuda") + (
|
||||
seq_len_kv - seq_len
|
||||
)
|
||||
else:
|
||||
ks, ke = _packed_requests_test_data(seq_len, seq_len_kv)
|
||||
|
||||
q_fp8 = q.to(torch.float8_e4m3fn)
|
||||
kv_fp8, kv_sf = per_custom_dims_cast_to_fp8(kv, (0,), True)
|
||||
q_scale, kernel_weights = mqa_logits_mx_inputs(q_fp8, None, weights)
|
||||
max_seqlen_k = int((ke - ks).max())
|
||||
logits = fp8_fp4_mqa_logits(
|
||||
(q_fp8, q_scale),
|
||||
(kv_fp8, encode_indexer_scales(kv_sf)),
|
||||
kernel_weights,
|
||||
ks,
|
||||
ke,
|
||||
max_seqlen_k,
|
||||
)
|
||||
assert logits.shape == (seq_len, max_seqlen_k)
|
||||
logits = as_fp32_logits(logits, ke - ks)
|
||||
|
||||
ref_logits = _ref_fp8_mqa_logits(
|
||||
q=q,
|
||||
kv=kv,
|
||||
weights=kernel_weights.float(),
|
||||
cu_seqlen_ks=ks,
|
||||
cu_seqlen_ke=ke,
|
||||
)
|
||||
# Compressed row i holds K positions [ks[i], ke[i]) from column zero.
|
||||
cols = ks[:, None] + torch.arange(max_seqlen_k, device="cuda")[None, :]
|
||||
valid = cols < ke[:, None]
|
||||
ref_logits = ref_logits.gather(1, cols.clamp(max=seq_len_kv - 1).long())
|
||||
ref_logits = ref_logits.masked_fill(~valid, 0)
|
||||
logits = logits.masked_fill(~valid, 0)
|
||||
diff = calc_diff(logits, ref_logits)
|
||||
assert diff < 1e-3, f"{diff=}"
|
||||
|
||||
|
||||
def _ref_fp8_fp4_paged_mqa_logits(
|
||||
q: torch.Tensor,
|
||||
kv_cache: torch.Tensor,
|
||||
weights: torch.Tensor,
|
||||
context_lens: torch.Tensor,
|
||||
block_tables: torch.Tensor,
|
||||
max_model_len: int,
|
||||
):
|
||||
batch_size, next_n, _, _ = q.size()
|
||||
_, block_size, _, _ = kv_cache.size()
|
||||
logits = torch.full(
|
||||
[batch_size * next_n, max_model_len],
|
||||
float("-inf"),
|
||||
device=q.device,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
context_lens_list = context_lens.tolist()
|
||||
for i in range(batch_size):
|
||||
context_len = context_lens_list[i]
|
||||
q_offsets = torch.arange(context_len - next_n, context_len, device="cuda")
|
||||
weight_slice = (
|
||||
weights[i * next_n : (i + 1) * next_n, :].transpose(0, 1).contiguous()
|
||||
)
|
||||
for block_rk in range(cdiv(context_len, block_size)):
|
||||
block_idx = block_tables[i][block_rk]
|
||||
qx, kx = q[i], kv_cache[block_idx]
|
||||
k_offsets = torch.arange(
|
||||
block_rk * block_size,
|
||||
(block_rk + 1) * block_size,
|
||||
device="cuda",
|
||||
)
|
||||
mask = (k_offsets[None, :] < context_len) & (
|
||||
k_offsets[None, :] <= q_offsets[:, None]
|
||||
)
|
||||
s = torch.where(
|
||||
mask[None, :, :],
|
||||
(qx.transpose(0, 1) @ kx.transpose(0, 1).transpose(1, 2)).to(
|
||||
logits.dtype
|
||||
),
|
||||
float("-inf"),
|
||||
)
|
||||
s = torch.relu(s) * weight_slice[..., None]
|
||||
s = s.sum(dim=0)
|
||||
logits[
|
||||
i * next_n : (i + 1) * next_n,
|
||||
block_rk * block_size : (block_rk + 1) * block_size,
|
||||
] = torch.where(k_offsets[None, :] <= q_offsets[:, None], s, float("-inf"))
|
||||
return logits
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
|
||||
@pytest.mark.skipif(not has_deep_gemm(), reason="DeepGEMM not available")
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.has_device_capability(90), reason="SM90 and SM100 only"
|
||||
)
|
||||
# next_n = 1 + num_speculative_tokens, so next_n=4 is MTP=3 (issue #35878).
|
||||
@pytest.mark.parametrize("batch_size,next_n", [(4, 1), (2, 2), (2, 4)])
|
||||
def test_deepgemm_fp8_fp4_paged_mqa_logits(batch_size: int, next_n: int):
|
||||
if not native_next_n_supported(next_n):
|
||||
pytest.skip(f"next_n={next_n} has no native kernel on this architecture")
|
||||
|
||||
torch.manual_seed(0)
|
||||
random.seed(0)
|
||||
|
||||
max_model_len = 4096
|
||||
for heads, index_dim in [(32, 128)]:
|
||||
for avg_kv in (2048,):
|
||||
num_blocks, blocksize = max_model_len * 2, 64
|
||||
|
||||
q = torch.randn(
|
||||
(batch_size, next_n, heads, index_dim),
|
||||
device="cuda",
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
kv_cache = torch.randn(
|
||||
(num_blocks, blocksize, 1, index_dim),
|
||||
device="cuda",
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
weights = torch.randn(
|
||||
(batch_size * next_n, heads),
|
||||
device="cuda",
|
||||
dtype=torch.float32,
|
||||
)
|
||||
|
||||
context_lens = (
|
||||
torch.randint(int(0.8 * avg_kv), int(1.2 * avg_kv), (batch_size,))
|
||||
.cuda()
|
||||
.to(torch.int32)
|
||||
)
|
||||
max_block_len = (
|
||||
(context_lens.max().item() + blocksize - 1) // blocksize * blocksize
|
||||
)
|
||||
block_tables = torch.zeros(
|
||||
(batch_size, max_block_len),
|
||||
device="cuda",
|
||||
dtype=torch.int32,
|
||||
)
|
||||
|
||||
counter = 0
|
||||
block_idx_pool = list(range(num_blocks))
|
||||
random.shuffle(block_idx_pool)
|
||||
for i in range(batch_size):
|
||||
ctx_len = int(context_lens[i].item())
|
||||
for j in range((ctx_len + blocksize - 1) // blocksize):
|
||||
block_tables[i][j] = block_idx_pool[counter]
|
||||
counter += 1
|
||||
|
||||
q_fp8 = q.to(torch.float8_e4m3fn)
|
||||
kv_cache_fp8 = kv_cache_cast_to_fp8(kv_cache)
|
||||
|
||||
# deep_gemm paged MQA logits requires 2D context_lens of
|
||||
# shape (B, next_n) (csrc/apis/attention.hpp:332-335);
|
||||
# see indexer.py:607-608. For each batch/next_n token, the
|
||||
# effective context length is context_lens[b] - next_n + j + 1.
|
||||
next_n_arange = torch.arange(next_n, device="cuda", dtype=torch.int32)
|
||||
context_lens_2d = (
|
||||
context_lens.unsqueeze(-1) - next_n + 1 + next_n_arange
|
||||
).contiguous()
|
||||
q_scale, kernel_weights = mqa_logits_mx_inputs(q_fp8, None, weights)
|
||||
kernel_q = (q_fp8, q_scale)
|
||||
kernel_context_lens, kernel_block_tables = context_lens_2d, block_tables
|
||||
indices = None
|
||||
if mqa_logits_use_mx_scales():
|
||||
# SM100 only takes varlen rows: one Q token each, tied to its
|
||||
# request by `indices`.
|
||||
assert q_scale is not None
|
||||
kernel_q = (
|
||||
q_fp8.flatten(0, 1)[:, None],
|
||||
q_scale.flatten(0, 1)[:, None],
|
||||
)
|
||||
kernel_context_lens = context_lens_2d.view(-1, 1)
|
||||
kernel_block_tables = block_tables.repeat_interleave(next_n, dim=0)
|
||||
indices = torch.arange(batch_size, device="cuda", dtype=torch.int32)
|
||||
indices = indices.repeat_interleave(next_n)
|
||||
schedule_metadata = get_paged_mqa_logits_metadata(
|
||||
kernel_context_lens, blocksize, get_num_sms(), indices=indices
|
||||
)
|
||||
logits = fp8_fp4_paged_mqa_logits(
|
||||
kernel_q,
|
||||
kv_cache_fp8,
|
||||
kernel_weights,
|
||||
kernel_context_lens,
|
||||
kernel_block_tables,
|
||||
schedule_metadata,
|
||||
max_model_len,
|
||||
indices=indices,
|
||||
)
|
||||
if indices is not None:
|
||||
logits = as_fp32_logits(logits, kernel_context_lens.view(-1))
|
||||
|
||||
ref_logits = _ref_fp8_fp4_paged_mqa_logits(
|
||||
q,
|
||||
kv_cache,
|
||||
kernel_weights.float(),
|
||||
context_lens,
|
||||
block_tables,
|
||||
max_model_len,
|
||||
)
|
||||
|
||||
positions = (
|
||||
torch.arange(max_model_len, device="cuda")
|
||||
.unsqueeze(0)
|
||||
.expand(batch_size * next_n, -1)
|
||||
)
|
||||
row_indices = torch.arange(batch_size * next_n, device="cuda") // next_n
|
||||
next_n_offset = torch.arange(batch_size * next_n, device="cuda") % next_n
|
||||
mask = positions <= (
|
||||
context_lens[row_indices] - next_n + next_n_offset
|
||||
).unsqueeze(1)
|
||||
|
||||
logits = logits.float().masked_fill(~mask, 0)
|
||||
ref_logits = ref_logits.masked_fill(~mask, 0)
|
||||
diff = calc_diff(logits, ref_logits)
|
||||
assert diff < 1e-3, f"{diff=}"
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
|
||||
@pytest.mark.skipif(not has_deep_gemm(), reason="DeepGEMM not available")
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.has_device_capability(90), reason="SM90 and SM100 only"
|
||||
)
|
||||
def test_deepgemm_paged_mqa_packed_manager_block_stride():
|
||||
torch.manual_seed(0)
|
||||
num_blocks, block_size, packed_rows = 4, 256, 320
|
||||
num_heads, head_dim, row_bytes = 64, 128, 132
|
||||
page_size, pages_per_block = 64, block_size // 64
|
||||
dense_stride, packed_stride = pages_per_block, packed_rows // page_size
|
||||
compact_pages = kv_cache_cast_to_fp8(
|
||||
torch.randn(
|
||||
num_blocks, block_size, 1, head_dim, device="cuda", dtype=torch.bfloat16
|
||||
).view(num_blocks * pages_per_block, page_size, 1, head_dim)
|
||||
)
|
||||
packed_backing = torch.zeros(
|
||||
num_blocks,
|
||||
packed_rows,
|
||||
row_bytes,
|
||||
device="cuda",
|
||||
dtype=torch.uint8,
|
||||
)
|
||||
packed_manager = packed_backing[:, :block_size]
|
||||
packed_manager.copy_(compact_pages.squeeze(2).view(num_blocks, block_size, -1))
|
||||
packed_pages = packed_backing.view(-1, page_size, row_bytes).unsqueeze(2)
|
||||
offsets = torch.arange(pages_per_block, device="cuda", dtype=torch.int32)
|
||||
tables = [
|
||||
(stride + offsets).unsqueeze(0) for stride in (dense_stride, packed_stride)
|
||||
]
|
||||
|
||||
q = torch.randn(1, 1, num_heads, head_dim, device="cuda", dtype=torch.bfloat16).to(
|
||||
torch.float8_e4m3fn
|
||||
)
|
||||
q_scale, weights = mqa_logits_mx_inputs(
|
||||
q, None, torch.randn(1, num_heads, device="cuda", dtype=torch.float32)
|
||||
)
|
||||
context_lens = torch.full((1, 1), block_size, device="cuda", dtype=torch.int32)
|
||||
indices = (
|
||||
torch.zeros(1, device="cuda", dtype=torch.int32)
|
||||
if mqa_logits_use_mx_scales()
|
||||
else None
|
||||
)
|
||||
schedule = get_paged_mqa_logits_metadata(
|
||||
context_lens, page_size, get_num_sms(), indices=indices
|
||||
)
|
||||
logits = [
|
||||
fp8_fp4_paged_mqa_logits(
|
||||
(q, q_scale),
|
||||
cache,
|
||||
weights,
|
||||
context_lens,
|
||||
table,
|
||||
schedule,
|
||||
block_size,
|
||||
indices=indices,
|
||||
)
|
||||
for cache, table in zip((compact_pages, packed_pages), tables)
|
||||
]
|
||||
torch.testing.assert_close(*logits, rtol=0, atol=0)
|
||||
@@ -1,737 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.v1.kv_cache_layout import KVCacheLayout
|
||||
|
||||
try:
|
||||
import flashinfer
|
||||
except ImportError:
|
||||
if current_platform.is_rocm():
|
||||
pytest.skip(
|
||||
"flashinfer is not supported for vLLM on ROCm.", allow_module_level=True
|
||||
)
|
||||
|
||||
import torch
|
||||
|
||||
NUM_HEADS = [(32, 8), (6, 1)]
|
||||
HEAD_SIZES = [128, 256]
|
||||
BLOCK_SIZES = [16, 32]
|
||||
DTYPES = [torch.bfloat16]
|
||||
NUM_BLOCKS = 32768 # Large enough to test overflow in index calculation.
|
||||
SOFT_CAPS = [None, 30.0]
|
||||
SLIDING_WINDOWS = [None, 64]
|
||||
|
||||
|
||||
def ref_paged_attn(
|
||||
query: torch.Tensor,
|
||||
key_cache: torch.Tensor,
|
||||
value_cache: torch.Tensor,
|
||||
query_lens: list[int],
|
||||
kv_lens: list[int],
|
||||
block_tables: torch.Tensor,
|
||||
scale: float,
|
||||
sliding_window: int | None = None,
|
||||
soft_cap: float | None = None,
|
||||
) -> torch.Tensor:
|
||||
num_seqs = len(query_lens)
|
||||
block_tables = block_tables.cpu().numpy()
|
||||
_, block_size, num_kv_heads, head_size = key_cache.shape
|
||||
|
||||
outputs: list[torch.Tensor] = []
|
||||
start_idx = 0
|
||||
for i in range(num_seqs):
|
||||
query_len = query_lens[i]
|
||||
kv_len = kv_lens[i]
|
||||
q = query[start_idx : start_idx + query_len]
|
||||
q *= scale
|
||||
|
||||
num_kv_blocks = (kv_len + block_size - 1) // block_size
|
||||
block_indices = block_tables[i, :num_kv_blocks]
|
||||
|
||||
k = key_cache[block_indices].view(-1, num_kv_heads, head_size)
|
||||
k = k[:kv_len]
|
||||
v = value_cache[block_indices].view(-1, num_kv_heads, head_size)
|
||||
v = v[:kv_len]
|
||||
|
||||
if q.shape[1] != k.shape[1]:
|
||||
k = torch.repeat_interleave(k, q.shape[1] // k.shape[1], dim=1)
|
||||
v = torch.repeat_interleave(v, q.shape[1] // v.shape[1], dim=1)
|
||||
attn = torch.einsum("qhd,khd->hqk", q, k).float()
|
||||
empty_mask = torch.ones(query_len, kv_len)
|
||||
mask = torch.triu(empty_mask, diagonal=kv_len - query_len + 1).bool()
|
||||
if sliding_window is not None:
|
||||
sliding_window_mask = (
|
||||
torch.triu(
|
||||
empty_mask, diagonal=kv_len - (query_len + sliding_window) + 1
|
||||
)
|
||||
.bool()
|
||||
.logical_not()
|
||||
)
|
||||
mask |= sliding_window_mask
|
||||
if soft_cap is not None:
|
||||
attn = soft_cap * torch.tanh(attn / soft_cap)
|
||||
attn.masked_fill_(mask, float("-inf"))
|
||||
attn = torch.softmax(attn, dim=-1).to(v.dtype)
|
||||
out = torch.einsum("hqk,khd->qhd", attn, v)
|
||||
|
||||
outputs.append(out)
|
||||
start_idx += query_len
|
||||
|
||||
return torch.cat(outputs, dim=0)
|
||||
|
||||
|
||||
def _make_paged_kv_metadata(
|
||||
kv_lens: list[int],
|
||||
block_size: int,
|
||||
num_blocks: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Build paged-KV metadata tensors for fast_plan_decode tests.
|
||||
|
||||
Returns:
|
||||
kv_indptr – CPU int32, shape [num_seqs + 1]
|
||||
kv_indices – CUDA int32, shape [total_blocks]
|
||||
kv_last_page_lens – CPU int32, shape [num_seqs]
|
||||
block_tables – CUDA int32, shape [num_seqs, max_blocks_per_seq]
|
||||
|
||||
"""
|
||||
num_seqs = len(kv_lens)
|
||||
max_blocks = (max(kv_lens) + block_size - 1) // block_size
|
||||
block_tables = torch.randint(
|
||||
0, num_blocks, (num_seqs, max_blocks), dtype=torch.int32, device="cuda"
|
||||
)
|
||||
|
||||
indptr_list = [0]
|
||||
indices_list: list[int] = []
|
||||
last_lens_list: list[int] = []
|
||||
for i, seq_len in enumerate(kv_lens):
|
||||
n = (seq_len + block_size - 1) // block_size
|
||||
indices_list.extend(block_tables[i, :n].cpu().tolist())
|
||||
indptr_list.append(indptr_list[-1] + n)
|
||||
last_lens_list.append(seq_len % block_size or block_size)
|
||||
|
||||
return (
|
||||
torch.tensor(indptr_list, dtype=torch.int32, device="cpu"),
|
||||
torch.tensor(indices_list, dtype=torch.int32, device="cuda"),
|
||||
torch.tensor(last_lens_list, dtype=torch.int32, device="cpu"),
|
||||
block_tables,
|
||||
)
|
||||
|
||||
|
||||
def _make_cg_decode_wrapper(
|
||||
num_seqs: int,
|
||||
kv_indices_buffer: torch.Tensor,
|
||||
workspace_buffer: torch.Tensor,
|
||||
use_tensor_cores: bool = True,
|
||||
) -> "flashinfer.BatchDecodeWithPagedKVCacheWrapper":
|
||||
"""Create a cudagraph-enabled BatchDecodeWithPagedKVCacheWrapper.
|
||||
|
||||
*kv_indices_buffer* is shared with the caller so that fast_plan_decode
|
||||
can avoid the device-to-device index copy on subsequent (cudagraph) calls.
|
||||
"""
|
||||
return flashinfer.BatchDecodeWithPagedKVCacheWrapper(
|
||||
workspace_buffer,
|
||||
"NHD",
|
||||
use_cuda_graph=True,
|
||||
paged_kv_indptr_buffer=torch.zeros(
|
||||
num_seqs + 1, dtype=torch.int32, device="cuda"
|
||||
),
|
||||
paged_kv_indices_buffer=kv_indices_buffer,
|
||||
paged_kv_last_page_len_buffer=torch.zeros(
|
||||
num_seqs, dtype=torch.int32, device="cuda"
|
||||
),
|
||||
use_tensor_cores=use_tensor_cores,
|
||||
)
|
||||
|
||||
|
||||
def test_fast_decode_plan_importable() -> None:
|
||||
"""fast_decode_plan must be importable from flashinfer.decode.
|
||||
|
||||
This is a forward-compatibility smoke test: if FlashInfer reorganises its
|
||||
public API the import will fail before any other test does.
|
||||
"""
|
||||
from flashinfer.decode import fast_decode_plan # noqa: F401
|
||||
|
||||
assert callable(fast_decode_plan)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@torch.inference_mode
|
||||
def test_fast_plan_decode_warmup_uses_full_plan(dtype: torch.dtype) -> None:
|
||||
"""On the first call fast_plan_decode must route through self.plan() and
|
||||
flip vllm_first_call to False on the wrapper object."""
|
||||
from unittest.mock import patch
|
||||
|
||||
from vllm.v1.attention.backends.flashinfer import fast_plan_decode
|
||||
|
||||
torch.set_default_device("cuda")
|
||||
set_random_seed(0)
|
||||
|
||||
kv_lens = [128, 64]
|
||||
block_size = 16
|
||||
num_seqs = len(kv_lens)
|
||||
num_query_heads, num_kv_heads = 8, 2
|
||||
head_size = 128
|
||||
|
||||
kv_indptr, kv_indices, kv_last_page_lens, _ = _make_paged_kv_metadata(
|
||||
kv_lens, block_size, NUM_BLOCKS
|
||||
)
|
||||
|
||||
workspace = torch.empty(128 * 1024 * 1024, dtype=torch.int8)
|
||||
wrapper = _make_cg_decode_wrapper(num_seqs, kv_indices.clone(), workspace)
|
||||
|
||||
assert getattr(wrapper, "vllm_first_call", True) is True
|
||||
|
||||
with patch.object(wrapper, "plan", wraps=wrapper.plan) as mock_plan:
|
||||
fast_plan_decode(
|
||||
wrapper,
|
||||
indptr_cpu=kv_indptr,
|
||||
indices=kv_indices,
|
||||
last_page_len_cpu=kv_last_page_lens,
|
||||
seq_lens_cpu=torch.tensor(kv_lens, dtype=torch.int32, device="cpu"),
|
||||
num_qo_heads=num_query_heads,
|
||||
num_kv_heads=num_kv_heads,
|
||||
head_dim=head_size,
|
||||
page_size=block_size,
|
||||
q_data_type=dtype,
|
||||
kv_data_type=dtype,
|
||||
)
|
||||
mock_plan.assert_called_once()
|
||||
|
||||
assert wrapper.vllm_first_call is False, (
|
||||
"vllm_first_call should be False after the first fast_plan_decode call"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kv_lens", [[1328, 18, 463], [1, 54, 293, 70]])
|
||||
@pytest.mark.parametrize("num_heads", NUM_HEADS)
|
||||
@pytest.mark.parametrize("head_size", HEAD_SIZES)
|
||||
@pytest.mark.parametrize("block_size", BLOCK_SIZES)
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@torch.inference_mode
|
||||
def test_fast_plan_decode_matches_full_plan(
|
||||
kv_lens: list[int],
|
||||
num_heads: tuple[int, int],
|
||||
head_size: int,
|
||||
block_size: int,
|
||||
dtype: torch.dtype,
|
||||
) -> None:
|
||||
"""fast_plan_decode's cudagraph path (delegating to FlashInfer's
|
||||
fast_decode_plan) must produce attention output numerically identical to
|
||||
a standard plan() call.
|
||||
|
||||
Both the warmup call (self.plan) and the subsequent fast call
|
||||
(fast_decode_plan) are verified against the same reference.
|
||||
"""
|
||||
from vllm.v1.attention.backends.flashinfer import fast_plan_decode
|
||||
|
||||
torch.set_default_device("cuda")
|
||||
set_random_seed(0)
|
||||
num_seqs = len(kv_lens)
|
||||
num_query_heads, num_kv_heads = num_heads
|
||||
|
||||
query = torch.randn(num_seqs, num_query_heads, head_size, dtype=dtype)
|
||||
key_value_cache = torch.randn(
|
||||
NUM_BLOCKS, 2, block_size, num_kv_heads, head_size, dtype=dtype
|
||||
)
|
||||
|
||||
kv_indptr, kv_indices, kv_last_page_lens, _ = _make_paged_kv_metadata(
|
||||
kv_lens, block_size, NUM_BLOCKS
|
||||
)
|
||||
|
||||
# Reference output via the standard plan()
|
||||
workspace_ref = torch.empty(128 * 1024 * 1024, dtype=torch.int8)
|
||||
ref_wrapper = flashinfer.BatchDecodeWithPagedKVCacheWrapper(
|
||||
workspace_ref, "NHD", use_tensor_cores=True
|
||||
)
|
||||
ref_wrapper.plan(
|
||||
kv_indptr,
|
||||
kv_indices,
|
||||
kv_last_page_lens,
|
||||
num_query_heads,
|
||||
num_kv_heads,
|
||||
head_size,
|
||||
block_size,
|
||||
"NONE",
|
||||
q_data_type=dtype,
|
||||
kv_data_type=dtype,
|
||||
)
|
||||
ref_output = ref_wrapper.run(query, key_value_cache)
|
||||
|
||||
# CUDAGraph wrapper exercised through fast_plan_decode
|
||||
kv_indices_buf = kv_indices.clone()
|
||||
workspace_cg = torch.empty(128 * 1024 * 1024, dtype=torch.int8)
|
||||
cg_wrapper = _make_cg_decode_wrapper(num_seqs, kv_indices_buf, workspace_cg)
|
||||
|
||||
plan_kwargs: dict = dict(
|
||||
indptr_cpu=kv_indptr,
|
||||
indices=kv_indices_buf,
|
||||
last_page_len_cpu=kv_last_page_lens,
|
||||
seq_lens_cpu=torch.tensor(kv_lens, dtype=torch.int32, device="cpu"),
|
||||
num_qo_heads=num_query_heads,
|
||||
num_kv_heads=num_kv_heads,
|
||||
head_dim=head_size,
|
||||
page_size=block_size,
|
||||
q_data_type=dtype,
|
||||
kv_data_type=dtype,
|
||||
)
|
||||
|
||||
# First call – warmup path (routes through self.plan)
|
||||
fast_plan_decode(cg_wrapper, **plan_kwargs)
|
||||
warmup_output = cg_wrapper.run(query, key_value_cache)
|
||||
torch.testing.assert_close(warmup_output, ref_output, atol=1e-2, rtol=1e-2)
|
||||
|
||||
# Second call – fast path (routes through fast_decode_plan from FlashInfer)
|
||||
fast_plan_decode(cg_wrapper, **plan_kwargs)
|
||||
fast_output = cg_wrapper.run(query, key_value_cache)
|
||||
torch.testing.assert_close(fast_output, ref_output, atol=1e-2, rtol=1e-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kv_lens", [[1328, 18, 463], [1, 54, 293, 70]])
|
||||
@pytest.mark.parametrize("num_heads", NUM_HEADS)
|
||||
@pytest.mark.parametrize("head_size", HEAD_SIZES)
|
||||
@pytest.mark.parametrize("block_size", BLOCK_SIZES)
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("soft_cap", SOFT_CAPS)
|
||||
@pytest.mark.parametrize("sliding_window", SLIDING_WINDOWS)
|
||||
@torch.inference_mode
|
||||
def test_flashinfer_decode_with_paged_kv(
|
||||
kv_lens: list[int],
|
||||
num_heads: tuple[int, int],
|
||||
head_size: int,
|
||||
dtype: torch.dtype,
|
||||
block_size: int,
|
||||
soft_cap: float | None,
|
||||
sliding_window: int | None,
|
||||
) -> None:
|
||||
torch.set_default_device("cuda")
|
||||
set_random_seed(0)
|
||||
num_seqs = len(kv_lens)
|
||||
num_query_heads = num_heads[0]
|
||||
num_kv_heads = num_heads[1]
|
||||
assert num_query_heads % num_kv_heads == 0
|
||||
max_kv_len = max(kv_lens)
|
||||
scale = head_size**-0.5
|
||||
|
||||
query = torch.randn(num_seqs, num_query_heads, head_size, dtype=dtype)
|
||||
|
||||
key_value_cache = torch.randn(
|
||||
NUM_BLOCKS, 2, block_size, num_kv_heads, head_size, dtype=dtype
|
||||
)
|
||||
key_cache = key_value_cache[:, 0, :, :, :].squeeze(1)
|
||||
value_cache = key_value_cache[:, 1, :, :, :].squeeze(1)
|
||||
|
||||
max_num_blocks_per_seq = (max_kv_len + block_size - 1) // block_size
|
||||
block_tables = torch.randint(
|
||||
0, NUM_BLOCKS, (num_seqs, max_num_blocks_per_seq), dtype=torch.int32
|
||||
)
|
||||
|
||||
kv_indptr = [0]
|
||||
kv_indices = []
|
||||
kv_last_page_lens = []
|
||||
for i in range(num_seqs):
|
||||
seq_len = kv_lens[i]
|
||||
assert seq_len > 0
|
||||
num_blocks = (seq_len + block_size - 1) // block_size
|
||||
kv_indices.extend(block_tables[i, :num_blocks])
|
||||
kv_indptr.append(kv_indptr[-1] + num_blocks)
|
||||
kv_last_page_len = seq_len % block_size
|
||||
if kv_last_page_len == 0:
|
||||
kv_last_page_len = block_size
|
||||
kv_last_page_lens.append(kv_last_page_len)
|
||||
|
||||
kv_indptr = torch.tensor(kv_indptr, dtype=torch.int32)
|
||||
kv_indices = torch.tensor(kv_indices, dtype=torch.int32)
|
||||
kv_last_page_lens = torch.tensor(kv_last_page_lens, dtype=torch.int32)
|
||||
|
||||
workspace_buffer = torch.empty(128 * 1024 * 1024, dtype=torch.int8)
|
||||
wrapper = flashinfer.BatchDecodeWithPagedKVCacheWrapper(
|
||||
workspace_buffer, "NHD", use_tensor_cores=True
|
||||
)
|
||||
wrapper.plan(
|
||||
kv_indptr,
|
||||
kv_indices,
|
||||
kv_last_page_lens,
|
||||
num_query_heads,
|
||||
num_kv_heads,
|
||||
head_size,
|
||||
block_size,
|
||||
"NONE",
|
||||
window_left=sliding_window - 1 if sliding_window is not None else -1,
|
||||
q_data_type=dtype,
|
||||
kv_data_type=dtype,
|
||||
logits_soft_cap=soft_cap,
|
||||
)
|
||||
|
||||
output = wrapper.run(query, key_value_cache)
|
||||
|
||||
ref_output = ref_paged_attn(
|
||||
query=query,
|
||||
key_cache=key_cache,
|
||||
value_cache=value_cache,
|
||||
query_lens=[1] * num_seqs,
|
||||
kv_lens=kv_lens,
|
||||
block_tables=block_tables,
|
||||
scale=scale,
|
||||
soft_cap=soft_cap,
|
||||
sliding_window=sliding_window,
|
||||
)
|
||||
(
|
||||
torch.testing.assert_close(output, ref_output, atol=1e-2, rtol=1e-2),
|
||||
f"{torch.max(torch.abs(output - ref_output))}",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("seq_lens", [[(1, 1328), (5, 18), (129, 463)]])
|
||||
@pytest.mark.parametrize("num_heads", NUM_HEADS)
|
||||
@pytest.mark.parametrize("head_size", HEAD_SIZES)
|
||||
@pytest.mark.parametrize("block_size", BLOCK_SIZES)
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("soft_cap", SOFT_CAPS)
|
||||
@pytest.mark.parametrize("sliding_window", SLIDING_WINDOWS)
|
||||
@torch.inference_mode
|
||||
def test_flashinfer_prefill_with_paged_kv(
|
||||
seq_lens: list[tuple[int, int]],
|
||||
num_heads: tuple[int, int],
|
||||
head_size: int,
|
||||
dtype: torch.dtype,
|
||||
block_size: int,
|
||||
soft_cap: float | None,
|
||||
sliding_window: int | None,
|
||||
) -> None:
|
||||
torch.set_default_device("cuda")
|
||||
set_random_seed(0)
|
||||
num_seqs = len(seq_lens)
|
||||
query_lens = [x[0] for x in seq_lens]
|
||||
kv_lens = [x[1] for x in seq_lens]
|
||||
num_query_heads = num_heads[0]
|
||||
num_kv_heads = num_heads[1]
|
||||
assert num_query_heads % num_kv_heads == 0
|
||||
max_kv_len = max(kv_lens)
|
||||
scale = head_size**-0.5
|
||||
|
||||
query = torch.randn(sum(query_lens), num_query_heads, head_size, dtype=dtype)
|
||||
key_value_cache = torch.randn(
|
||||
NUM_BLOCKS, 2, block_size, num_kv_heads, head_size, dtype=dtype
|
||||
)
|
||||
key_cache = key_value_cache[:, 0, :, :, :].squeeze(1)
|
||||
value_cache = key_value_cache[:, 1, :, :, :].squeeze(1)
|
||||
|
||||
# Normalize the scale of the key and value caches to mitigate
|
||||
# numerical instability.
|
||||
key_cache /= head_size**0.5
|
||||
value_cache /= head_size**0.5
|
||||
|
||||
max_num_blocks_per_seq = (max_kv_len + block_size - 1) // block_size
|
||||
block_tables = torch.randint(
|
||||
0, NUM_BLOCKS, (num_seqs, max_num_blocks_per_seq), dtype=torch.int32
|
||||
)
|
||||
|
||||
qo_indptr = [0]
|
||||
kv_indptr = [0]
|
||||
kv_indices = []
|
||||
kv_last_page_lens = []
|
||||
for i in range(num_seqs):
|
||||
seq_len = kv_lens[i]
|
||||
assert seq_len > 0
|
||||
num_blocks = (seq_len + block_size - 1) // block_size
|
||||
kv_indices.extend(block_tables[i, :num_blocks])
|
||||
kv_indptr.append(kv_indptr[-1] + num_blocks)
|
||||
kv_last_page_len = seq_len % block_size
|
||||
if kv_last_page_len == 0:
|
||||
kv_last_page_len = block_size
|
||||
kv_last_page_lens.append(kv_last_page_len)
|
||||
qo_indptr.append(qo_indptr[-1] + query_lens[i])
|
||||
|
||||
qo_indptr = torch.tensor(qo_indptr, dtype=torch.int32)
|
||||
kv_indptr = torch.tensor(kv_indptr, dtype=torch.int32)
|
||||
kv_indices = torch.tensor(kv_indices, dtype=torch.int32)
|
||||
kv_last_page_lens = torch.tensor(kv_last_page_lens, dtype=torch.int32)
|
||||
|
||||
workspace_buffer = torch.empty(128 * 1024 * 1024, dtype=torch.int8)
|
||||
wrapper = flashinfer.BatchPrefillWithPagedKVCacheWrapper(workspace_buffer, "NHD")
|
||||
wrapper.plan(
|
||||
qo_indptr,
|
||||
kv_indptr,
|
||||
kv_indices,
|
||||
kv_last_page_lens,
|
||||
num_query_heads,
|
||||
num_kv_heads,
|
||||
head_size,
|
||||
block_size,
|
||||
window_left=sliding_window - 1 if sliding_window is not None else -1,
|
||||
q_data_type=dtype,
|
||||
kv_data_type=dtype,
|
||||
logits_soft_cap=soft_cap,
|
||||
)
|
||||
|
||||
output = wrapper.run(
|
||||
query,
|
||||
key_value_cache,
|
||||
)
|
||||
|
||||
ref_output = ref_paged_attn(
|
||||
query=query,
|
||||
key_cache=key_cache,
|
||||
value_cache=value_cache,
|
||||
query_lens=query_lens,
|
||||
kv_lens=kv_lens,
|
||||
block_tables=block_tables,
|
||||
scale=scale,
|
||||
soft_cap=soft_cap,
|
||||
sliding_window=sliding_window,
|
||||
)
|
||||
(
|
||||
torch.testing.assert_close(output, ref_output, atol=5e-2, rtol=1e-2),
|
||||
f"{torch.max(torch.abs(output - ref_output))}",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("seq_lens", [[(1, 132), (5, 18)]])
|
||||
@pytest.mark.parametrize("num_heads", NUM_HEADS)
|
||||
@pytest.mark.parametrize("head_size", HEAD_SIZES)
|
||||
@pytest.mark.parametrize("block_size", BLOCK_SIZES)
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("soft_cap", SOFT_CAPS)
|
||||
def test_flashinfer_prefill_with_paged_fp8_kv(
|
||||
seq_lens: list[tuple[int, int]],
|
||||
num_heads: tuple[int, int],
|
||||
head_size: int,
|
||||
dtype: torch.dtype,
|
||||
block_size: int,
|
||||
soft_cap: float | None,
|
||||
) -> None:
|
||||
pytest.skip("TODO: fix the accuracy issue")
|
||||
torch.set_default_device("cuda")
|
||||
set_random_seed(0)
|
||||
num_seqs = len(seq_lens)
|
||||
query_lens = [x[0] for x in seq_lens]
|
||||
kv_lens = [x[1] for x in seq_lens]
|
||||
num_query_heads = num_heads[0]
|
||||
num_kv_heads = num_heads[1]
|
||||
assert num_query_heads % num_kv_heads == 0
|
||||
max_kv_len = max(kv_lens)
|
||||
scale = head_size**-0.5
|
||||
|
||||
kv_cache_dtype = torch.float8_e4m3fn
|
||||
|
||||
query = torch.randn(sum(query_lens), num_query_heads, head_size, dtype=dtype)
|
||||
NUM_BLOCKS_FP8 = 2048
|
||||
key_value_cache = torch.randn(
|
||||
NUM_BLOCKS_FP8, 2, block_size, num_kv_heads, head_size, dtype=dtype
|
||||
)
|
||||
key_cache, value_cache = torch.chunk(key_value_cache, 2, dim=1)
|
||||
key_cache /= head_size**0.5
|
||||
value_cache /= head_size**0.5
|
||||
|
||||
k_scale = key_cache.amax().item() / 448.0
|
||||
v_scale = value_cache.amax().item() / 448.0
|
||||
|
||||
kv_cache_fp8 = torch.cat([key_cache / k_scale, value_cache / v_scale], dim=1).to(
|
||||
kv_cache_dtype
|
||||
)
|
||||
|
||||
assert kv_cache_fp8.shape == key_value_cache.shape
|
||||
max_num_blocks_per_seq = (max_kv_len + block_size - 1) // block_size
|
||||
block_tables = torch.randint(
|
||||
0, NUM_BLOCKS_FP8, (num_seqs, max_num_blocks_per_seq), dtype=torch.int32
|
||||
)
|
||||
|
||||
qo_indptr = [0]
|
||||
kv_indptr = [0]
|
||||
kv_indices = []
|
||||
kv_last_page_lens = []
|
||||
for i in range(num_seqs):
|
||||
seq_len = kv_lens[i]
|
||||
assert seq_len > 0
|
||||
num_blocks = (seq_len + block_size - 1) // block_size
|
||||
kv_indices.extend(block_tables[i, :num_blocks])
|
||||
kv_indptr.append(kv_indptr[-1] + num_blocks)
|
||||
kv_last_page_len = seq_len % block_size
|
||||
if kv_last_page_len == 0:
|
||||
kv_last_page_len = block_size
|
||||
kv_last_page_lens.append(kv_last_page_len)
|
||||
qo_indptr.append(qo_indptr[-1] + query_lens[i])
|
||||
|
||||
qo_indptr = torch.tensor(qo_indptr, dtype=torch.int32)
|
||||
kv_indptr = torch.tensor(kv_indptr, dtype=torch.int32)
|
||||
kv_indices = torch.tensor(kv_indices, dtype=torch.int32)
|
||||
kv_last_page_lens = torch.tensor(kv_last_page_lens, dtype=torch.int32)
|
||||
|
||||
workspace_buffer = torch.empty(128 * 1024 * 1024, dtype=torch.int8)
|
||||
wrapper = flashinfer.BatchPrefillWithPagedKVCacheWrapper(workspace_buffer, "NHD")
|
||||
wrapper.plan(
|
||||
qo_indptr,
|
||||
kv_indptr,
|
||||
kv_indices,
|
||||
kv_last_page_lens,
|
||||
num_query_heads,
|
||||
num_kv_heads,
|
||||
head_size,
|
||||
block_size,
|
||||
q_data_type=dtype,
|
||||
kv_data_type=kv_cache_dtype,
|
||||
logits_soft_cap=soft_cap,
|
||||
)
|
||||
|
||||
output = wrapper.run(query, kv_cache_fp8, k_scale=k_scale, v_scale=v_scale)
|
||||
|
||||
ref_output = ref_paged_attn(
|
||||
query=query,
|
||||
key_cache=key_cache.squeeze(1),
|
||||
value_cache=value_cache.squeeze(1),
|
||||
query_lens=query_lens,
|
||||
kv_lens=kv_lens,
|
||||
block_tables=block_tables,
|
||||
scale=scale,
|
||||
soft_cap=soft_cap,
|
||||
)
|
||||
del query
|
||||
del block_tables
|
||||
# verify prefill fp8
|
||||
(
|
||||
torch.testing.assert_close(output, ref_output, atol=5e-2, rtol=1e-2),
|
||||
f"{torch.max(torch.abs(output - ref_output))}",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kv_lens", [[1328, 18, 463], [1, 54, 293, 70]])
|
||||
@pytest.mark.parametrize("num_heads", NUM_HEADS)
|
||||
@pytest.mark.parametrize("head_size", HEAD_SIZES)
|
||||
@pytest.mark.parametrize("block_size", BLOCK_SIZES)
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("soft_cap", SOFT_CAPS)
|
||||
@pytest.mark.skip(reason="TODO: fix the accuracy issue")
|
||||
@torch.inference_mode
|
||||
def test_flashinfer_decode_with_paged_fp8_kv(
|
||||
kv_lens: list[int],
|
||||
num_heads: tuple[int, int],
|
||||
head_size: int,
|
||||
dtype: torch.dtype,
|
||||
block_size: int,
|
||||
soft_cap: float | None,
|
||||
) -> None:
|
||||
# test doesn't work for num_heads = (16,16)
|
||||
torch.set_default_device("cuda")
|
||||
set_random_seed(0)
|
||||
num_seqs = len(kv_lens)
|
||||
num_query_heads = num_heads[0]
|
||||
num_kv_heads = num_heads[1]
|
||||
assert num_query_heads % num_kv_heads == 0
|
||||
max_kv_len = max(kv_lens)
|
||||
scale = head_size**-0.5
|
||||
use_tensor_cores = True
|
||||
kv_cache_dtype = torch.float8_e4m3fn
|
||||
|
||||
query = torch.randn(num_seqs, num_query_heads, head_size, dtype=dtype)
|
||||
NUM_BLOCKS_FP8 = 2048
|
||||
key_value_cache = torch.randn(
|
||||
NUM_BLOCKS_FP8, 2, block_size, num_kv_heads, head_size, dtype=dtype
|
||||
)
|
||||
key_cache, value_cache = torch.chunk(key_value_cache, 2, dim=1)
|
||||
key_cache /= head_size**0.5
|
||||
value_cache /= head_size**0.5
|
||||
|
||||
k_scale = key_cache.amax().item() / 448.0
|
||||
v_scale = value_cache.amax().item() / 448.0
|
||||
|
||||
key_cache_fp8 = (key_cache / k_scale).to(kv_cache_dtype)
|
||||
value_cache_fp8 = (value_cache / v_scale).to(kv_cache_dtype)
|
||||
assert key_cache_fp8.shape[1] == 1 and value_cache_fp8.shape[1] == 1
|
||||
kv_cache_fp8 = torch.cat([key_cache_fp8, value_cache_fp8], dim=1)
|
||||
|
||||
max_num_blocks_per_seq = (max_kv_len + block_size - 1) // block_size
|
||||
block_tables = torch.randint(
|
||||
0, NUM_BLOCKS_FP8, (num_seqs, max_num_blocks_per_seq), dtype=torch.int32
|
||||
)
|
||||
|
||||
kv_indptr = [0]
|
||||
kv_indices = []
|
||||
kv_last_page_lens = []
|
||||
for i in range(num_seqs):
|
||||
seq_len = kv_lens[i]
|
||||
assert seq_len > 0
|
||||
num_blocks = (seq_len + block_size - 1) // block_size
|
||||
kv_indices.extend(block_tables[i, :num_blocks])
|
||||
kv_indptr.append(kv_indptr[-1] + num_blocks)
|
||||
kv_last_page_len = seq_len % block_size
|
||||
if kv_last_page_len == 0:
|
||||
kv_last_page_len = block_size
|
||||
kv_last_page_lens.append(kv_last_page_len)
|
||||
|
||||
kv_indptr = torch.tensor(kv_indptr, dtype=torch.int32)
|
||||
kv_indices = torch.tensor(kv_indices, dtype=torch.int32)
|
||||
kv_last_page_lens = torch.tensor(kv_last_page_lens, dtype=torch.int32)
|
||||
|
||||
workspace_buffer = torch.empty(128 * 1024 * 1024, dtype=torch.int8)
|
||||
wrapper = flashinfer.BatchDecodeWithPagedKVCacheWrapper(
|
||||
workspace_buffer, "NHD", use_tensor_cores=use_tensor_cores
|
||||
)
|
||||
wrapper.plan(
|
||||
kv_indptr,
|
||||
kv_indices,
|
||||
kv_last_page_lens,
|
||||
num_query_heads,
|
||||
num_kv_heads,
|
||||
head_size,
|
||||
block_size,
|
||||
"NONE",
|
||||
q_data_type=dtype,
|
||||
kv_data_type=kv_cache_dtype,
|
||||
logits_soft_cap=soft_cap,
|
||||
)
|
||||
output = wrapper.run(query, kv_cache_fp8, k_scale=k_scale, v_scale=v_scale)
|
||||
key_cache = key_value_cache[:, 0, :, :, :].squeeze(1)
|
||||
value_cache = key_value_cache[:, 1, :, :, :].squeeze(1)
|
||||
|
||||
ref_output = ref_paged_attn(
|
||||
query=query,
|
||||
key_cache=key_cache,
|
||||
value_cache=value_cache,
|
||||
query_lens=[1] * num_seqs,
|
||||
kv_lens=kv_lens,
|
||||
block_tables=block_tables,
|
||||
scale=scale,
|
||||
soft_cap=soft_cap,
|
||||
)
|
||||
# Temporary fix: Increasing the tolerance. Seems like a flashinfer issue
|
||||
(
|
||||
torch.testing.assert_close(output, ref_output, atol=2e-2, rtol=1e-2),
|
||||
f"{torch.max(torch.abs(output - ref_output))}",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not any(current_platform.is_device_capability_family(f) for f in (80, 90, 120)),
|
||||
reason="NVFP4 fa2 path",
|
||||
)
|
||||
@pytest.mark.parametrize("layout", ["LBNHC", "LBHNC", "BLNHC", "BLHNC"])
|
||||
def test_flashinfer_nvfp4_kv_cache_update(default_vllm_config, layout):
|
||||
from vllm.v1.attention.backends import flashinfer as fi
|
||||
|
||||
torch.set_default_device("cuda")
|
||||
order = KVCacheLayout[layout].stride_order
|
||||
# [L, B, 2H, N, 64 + 8] NVFP4 pages for two layers; layer 1 is written.
|
||||
shape = (2, 6, 4, 16, 64 + 8)
|
||||
layers = torch.zeros([shape[i] for i in order], dtype=torch.uint8)
|
||||
layers = layers.permute(*(order.index(i) for i in range(5)))
|
||||
one = torch.ones(())
|
||||
layer = torch.nn.Module()
|
||||
layer._k_scale = layer._v_scale = one
|
||||
kv = torch.randn(2, 96, 2, 128, dtype=torch.bfloat16)
|
||||
impl = fi.FlashInferImpl(4, 128, 1.0, 2, None, None, "nvfp4")
|
||||
impl.do_kv_cache_update(layer, kv[0], kv[1], layers[1], torch.arange(96))
|
||||
|
||||
data, scales, kv_layout = fi._nvfp4_kv_views(layers[1], 2)
|
||||
pages = torch.arange(6, dtype=torch.int32)[None]
|
||||
seq_lens = torch.tensor([96], dtype=torch.int32)
|
||||
deq = torch.empty_like(kv)[:, None]
|
||||
flashinfer.nvfp4_kv_dequantize_paged(
|
||||
data, scales, pages, seq_lens, one, one, *deq, kv_layout
|
||||
)
|
||||
assert ((deq[:, 0] - kv).norm() / kv.norm()) < 0.2
|
||||
assert not layers[0].any()
|
||||
@@ -1,389 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import Tensor
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
FLASHINFER_WORKSPACE_BUFFER_SIZE = 128 * 1024 * 1024
|
||||
|
||||
if not current_platform.is_cuda() or not current_platform.has_device_capability(90):
|
||||
pytest.skip(
|
||||
reason="FlashInfer MLA requires CUDA compute capability 9.0 or above.",
|
||||
allow_module_level=True,
|
||||
)
|
||||
else:
|
||||
from flashinfer.decode import trtllm_batch_decode_with_kv_cache_mla
|
||||
from flashinfer.mla import BatchMLAPagedAttentionWrapper
|
||||
|
||||
requires_sm90 = pytest.mark.skipif(
|
||||
not current_platform.is_device_capability_family(90),
|
||||
reason="This test requires an SM90 GPU.",
|
||||
)
|
||||
requires_sm10x = pytest.mark.skipif(
|
||||
not current_platform.is_device_capability_family(100),
|
||||
reason="This test requires an SM10x GPU.",
|
||||
)
|
||||
|
||||
# Deepseek R1 MLA config.
|
||||
NUM_HEADS = 128
|
||||
KV_LORA_RANK = 512
|
||||
QK_NOPE_HEAD_DIM = 128
|
||||
QK_ROPE_HEAD_DIM = 64
|
||||
QK_HEAD_DIM = KV_LORA_RANK + QK_ROPE_HEAD_DIM
|
||||
SCALE = (QK_NOPE_HEAD_DIM + QK_ROPE_HEAD_DIM) ** -0.5
|
||||
|
||||
|
||||
def _make_decode_inputs(bs: int, block_size: int, dtype: torch.dtype):
|
||||
"""Build valid trtllm MLA decode inputs on the current CUDA device."""
|
||||
max_seq_len_cap = 1024
|
||||
seq_lens = [torch.randint(2, max_seq_len_cap, (1,)).item() for _ in range(bs)]
|
||||
seq_lens[-1] = max_seq_len_cap
|
||||
max_seq_len = max(seq_lens)
|
||||
seq_lens_tensor = torch.tensor(seq_lens, dtype=torch.int32)
|
||||
|
||||
# Generate block tables with random but unique block IDs
|
||||
# From https://github.com/flashinfer-ai/flashinfer/pull/1222
|
||||
blocks_per_seq = (seq_lens_tensor + block_size - 1) // block_size
|
||||
max_num_blocks_per_seq = max(blocks_per_seq.max().item(), 4)
|
||||
total_blocks_needed = int(sum(blocks_per_seq))
|
||||
all_block_ids = torch.randperm(total_blocks_needed)
|
||||
|
||||
block_tables = torch.zeros((bs, max_num_blocks_per_seq), dtype=torch.int32)
|
||||
block_id = 0
|
||||
for i in range(bs):
|
||||
num_blocks_needed = blocks_per_seq[i]
|
||||
block_tables[i, :num_blocks_needed] = all_block_ids[
|
||||
block_id : block_id + num_blocks_needed
|
||||
]
|
||||
block_id += num_blocks_needed
|
||||
|
||||
kv_cache = torch.randn(block_tables.numel(), block_size, QK_HEAD_DIM).to(dtype)
|
||||
q = torch.randn(bs, NUM_HEADS, QK_HEAD_DIM).to(dtype)
|
||||
return q, kv_cache, block_tables, seq_lens_tensor, max_seq_len
|
||||
|
||||
|
||||
def ref_mla(
|
||||
out: Tensor, # (bs, num_heads, v_head_dim)
|
||||
query: Tensor, # (bs, num_heads, head_dim)
|
||||
kv_cache: Tensor, # (num_blocks, block_size, head_dim)
|
||||
scale: float,
|
||||
block_tables: Tensor, # (bs, max_num_blocks)
|
||||
seq_lens: Tensor, # (bs,)
|
||||
):
|
||||
bs, num_heads, v_head_dim = out.shape
|
||||
head_dim = query.shape[2]
|
||||
|
||||
for i in range(bs):
|
||||
# gather and flatten KV-cache
|
||||
kv = kv_cache[block_tables[i]] # (max_num_blocks, block_size, head_dim)
|
||||
kv = kv.view(1, -1, head_dim)[:, : seq_lens[i]] # (1, seq_len, head_dim)
|
||||
v = kv[:, :, :v_head_dim]
|
||||
|
||||
q = query[i].view(num_heads, 1, head_dim)
|
||||
o = F.scaled_dot_product_attention(q, kv, v, scale=scale, enable_gqa=True)
|
||||
out[i] = o.view(num_heads, v_head_dim)
|
||||
|
||||
return out
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16])
|
||||
@pytest.mark.parametrize("bs", [1, 2, 4, 16])
|
||||
@pytest.mark.parametrize("block_size", [32, 64])
|
||||
@requires_sm10x
|
||||
def test_flashinfer_mla_decode(dtype: torch.dtype, bs: int, block_size: int):
|
||||
torch.set_default_device("cuda")
|
||||
torch.manual_seed(42)
|
||||
|
||||
q, kv_cache, block_tables, seq_lens_tensor, max_seq_len = _make_decode_inputs(
|
||||
bs, block_size, dtype
|
||||
)
|
||||
|
||||
out_ref = q.new_zeros(bs, NUM_HEADS, KV_LORA_RANK)
|
||||
ref_mla(out_ref, q, kv_cache, SCALE, block_tables, seq_lens_tensor)
|
||||
|
||||
workspace_buffer = torch.zeros(
|
||||
FLASHINFER_WORKSPACE_BUFFER_SIZE,
|
||||
dtype=torch.uint8,
|
||||
device=q.device,
|
||||
)
|
||||
# Flashinfer MLA expects the query to be of shape
|
||||
# (bs, q_len_per_request, num_heads, qk_head_dim),
|
||||
# where q_len_per_request is the MTP query length (=1 without MTP)
|
||||
q = q.unsqueeze(1)
|
||||
|
||||
out_ans = trtllm_batch_decode_with_kv_cache_mla(
|
||||
query=q,
|
||||
kv_cache=kv_cache.unsqueeze(1),
|
||||
workspace_buffer=workspace_buffer,
|
||||
qk_nope_head_dim=QK_NOPE_HEAD_DIM,
|
||||
kv_lora_rank=KV_LORA_RANK,
|
||||
qk_rope_head_dim=QK_ROPE_HEAD_DIM,
|
||||
block_tables=block_tables,
|
||||
seq_lens=seq_lens_tensor,
|
||||
max_seq_len=max_seq_len,
|
||||
bmm1_scale=SCALE,
|
||||
)
|
||||
out_ans = out_ans.squeeze(1)
|
||||
torch.testing.assert_close(out_ans, out_ref, atol=1e-2, rtol=1e-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float8_e4m3fn])
|
||||
@pytest.mark.parametrize("mode", ["tp", "dcp", "hisparse"])
|
||||
@requires_sm10x
|
||||
def test_flashinfer_trtllm_sparse_mla_decode_without_rope(dtype, mode, monkeypatch):
|
||||
"""TP skips preparation; empty DCP contributions and HiSparse outputs vanish."""
|
||||
import vllm.v1.attention.backends.mla.flashinfer_mla_sparse as backend
|
||||
from vllm.v1.attention.backends.mla.index_group import HiSparseMLAIndexGroup
|
||||
|
||||
torch.set_default_device("cuda")
|
||||
torch.manual_seed(42)
|
||||
batch_size, num_heads, block_size, num_blocks, sparse_topk = 3, 8, 64, 4, 2176
|
||||
valid_lens = [1 if mode == "tp" else 0, 17, 73]
|
||||
counts = torch.tensor(valid_lens, dtype=torch.int32)
|
||||
query = torch.randn(batch_size, 1, num_heads, KV_LORA_RANK).to(dtype)
|
||||
kv_cache = torch.randn(num_blocks, block_size, KV_LORA_RANK).to(dtype)
|
||||
num_slots = num_blocks * block_size
|
||||
indices = torch.full((batch_size, sparse_topk), -1, dtype=torch.int32)
|
||||
for token, length in enumerate(valid_lens):
|
||||
indices[token, :length] = torch.randperm(num_slots)[:length].int()
|
||||
empty = counts == 0
|
||||
reference_indices = indices.clone()
|
||||
reference_indices[:, 0].masked_fill_(empty, 0)
|
||||
workspace = torch.empty(FLASHINFER_WORKSPACE_BUFFER_SIZE, dtype=torch.int8)
|
||||
|
||||
impl = object.__new__(backend.FlashInferMLASparseImpl)
|
||||
impl.qk_nope_head_dim = QK_NOPE_HEAD_DIM
|
||||
impl.kv_lora_rank = KV_LORA_RANK
|
||||
impl.qk_rope_head_dim = 0
|
||||
impl.index_group = (
|
||||
object.__new__(HiSparseMLAIndexGroup) if mode == "hisparse" else None
|
||||
)
|
||||
impl._workspace_buffer = workspace
|
||||
impl.bmm1_scale = QK_NOPE_HEAD_DIM**-0.5
|
||||
impl.bmm2_scale = 1.0
|
||||
impl.is_nope_mla = True
|
||||
impl.need_to_return_lse_for_decode = mode == "dcp"
|
||||
if mode == "tp":
|
||||
|
||||
def unexpected_helper(*args, **kwargs):
|
||||
pytest.fail("TP must not launch empty-query preparation")
|
||||
|
||||
monkeypatch.setattr(
|
||||
backend, "prepare_sparse_mla_safe_lengths", unexpected_helper
|
||||
)
|
||||
|
||||
out, lse = impl._forward_mqa_kernel(
|
||||
query.squeeze(1),
|
||||
kv_cache,
|
||||
indices,
|
||||
counts,
|
||||
layer=None,
|
||||
block_size=block_size,
|
||||
is_decode=True,
|
||||
)
|
||||
reference = trtllm_batch_decode_with_kv_cache_mla(
|
||||
query=query,
|
||||
kv_cache=kv_cache.unsqueeze(1),
|
||||
workspace_buffer=workspace,
|
||||
qk_nope_head_dim=QK_NOPE_HEAD_DIM,
|
||||
kv_lora_rank=KV_LORA_RANK,
|
||||
qk_rope_head_dim=0,
|
||||
block_tables=reference_indices.unsqueeze(1),
|
||||
seq_lens=counts,
|
||||
max_seq_len=sparse_topk,
|
||||
sparse_mla_top_k=sparse_topk,
|
||||
sparse_mla_top_k_lens=counts.clamp(min=1),
|
||||
bmm1_scale=QK_NOPE_HEAD_DIM**-0.5,
|
||||
bmm2_scale=1.0,
|
||||
return_lse=mode == "dcp",
|
||||
)
|
||||
if mode == "dcp":
|
||||
reference, reference_lse = reference
|
||||
reference_lse = reference_lse.reshape(batch_size, num_heads)
|
||||
reference_lse.masked_fill_(empty[:, None], float("-inf"))
|
||||
torch.testing.assert_close(lse, reference_lse, atol=0, rtol=0)
|
||||
|
||||
from vllm.v1.attention.ops.dcp import CPTritonContext, correct_attn_out
|
||||
|
||||
# The peer owns only our empty selections. Check our contribution after
|
||||
# the production merge, without zeroing the local attention output.
|
||||
peer_lse = torch.full_like(lse, float("-inf")).masked_fill(empty[:, None], 0)
|
||||
out, _ = correct_attn_out(
|
||||
out, torch.stack((lse, peer_lse)), cp_rank=0, ctx=CPTritonContext()
|
||||
)
|
||||
else:
|
||||
assert lse is None
|
||||
reference = reference.squeeze(1).masked_fill(empty[:, None, None], 0)
|
||||
torch.testing.assert_close(out, reference, atol=0, rtol=0)
|
||||
|
||||
if dtype == torch.bfloat16:
|
||||
flat_cache = kv_cache.view(num_slots, KV_LORA_RANK).float()
|
||||
refs = []
|
||||
for token, length in enumerate(valid_lens):
|
||||
selected_kv = flat_cache[reference_indices[token, :length].long()]
|
||||
scores = torch.einsum("hd,kd->hk", query[token, 0].float(), selected_kv)
|
||||
probs = torch.softmax(scores * QK_NOPE_HEAD_DIM**-0.5, dim=-1)
|
||||
refs.append(torch.einsum("hk,kd->hd", probs, selected_kv))
|
||||
ref = torch.stack(refs).to(torch.bfloat16)
|
||||
torch.testing.assert_close(out, ref, atol=2e-2, rtol=2e-2)
|
||||
|
||||
|
||||
@requires_sm10x
|
||||
def test_sparse_mla_safe_lengths():
|
||||
from vllm.v1.attention.backends.mla.sparse_utils import (
|
||||
prepare_sparse_mla_safe_lengths,
|
||||
)
|
||||
|
||||
device = "cuda"
|
||||
num_tokens = 259
|
||||
# Include a partial tile and expose accidental writes into adjacent storage.
|
||||
counts_storage = torch.full((num_tokens, 2), 17, dtype=torch.int32, device=device)
|
||||
counts = counts_storage[:, 0]
|
||||
counts[::3] = 0
|
||||
original_counts = counts_storage.clone()
|
||||
empty = counts == 0
|
||||
indices_storage = torch.full(
|
||||
(num_tokens, 256), 42, dtype=torch.int32, device=device
|
||||
)
|
||||
indices = indices_storage[:, ::2]
|
||||
expected_indices = indices_storage.clone()
|
||||
expected_indices[:, 0].masked_fill_(empty, 0)
|
||||
safe = prepare_sparse_mla_safe_lengths(indices, counts)
|
||||
torch.testing.assert_close(safe, counts.clamp(min=1), rtol=0, atol=0)
|
||||
torch.testing.assert_close(indices_storage, expected_indices, rtol=0, atol=0)
|
||||
torch.testing.assert_close(counts_storage, original_counts, rtol=0, atol=0)
|
||||
|
||||
|
||||
@requires_sm90
|
||||
def test_flashinfer_sm90_fp8_mla_decode_without_rope():
|
||||
"""Hopper FA3 supports BF16 queries over an FP8 cache without KPE."""
|
||||
torch.manual_seed(42)
|
||||
device = torch.device("cuda")
|
||||
batch_size = 2
|
||||
num_heads = 16
|
||||
page_size = 16
|
||||
num_pages = 6
|
||||
|
||||
q_nope = torch.randn(
|
||||
batch_size,
|
||||
num_heads,
|
||||
KV_LORA_RANK,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
q_pe = torch.empty(
|
||||
batch_size,
|
||||
num_heads,
|
||||
0,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
|
||||
ckv = torch.randn(
|
||||
num_pages,
|
||||
page_size,
|
||||
KV_LORA_RANK,
|
||||
device=device,
|
||||
)
|
||||
fp8_max = torch.finfo(torch.float8_e4m3fn).max
|
||||
ckv_scale = ckv.abs().max().item() / fp8_max
|
||||
ckv_fp8 = (ckv / ckv_scale).clamp(-fp8_max, fp8_max).to(torch.float8_e4m3fn)
|
||||
scale_bf16 = torch.tensor(ckv_scale, dtype=torch.bfloat16, device=device)
|
||||
ckv_ref = ckv_fp8.to(torch.bfloat16) * scale_bf16
|
||||
kpe_fp8 = torch.empty(
|
||||
num_pages,
|
||||
page_size,
|
||||
0,
|
||||
dtype=torch.float8_e4m3fn,
|
||||
device=device,
|
||||
)
|
||||
kpe_ref = torch.empty(
|
||||
num_pages,
|
||||
page_size,
|
||||
0,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
|
||||
qo_indptr = torch.tensor([0, 1, 2], dtype=torch.int32, device=device)
|
||||
kv_indptr = torch.tensor([0, 3, 5], dtype=torch.int32, device=device)
|
||||
kv_indices = torch.tensor([4, 1, 3, 0, 5], dtype=torch.int32, device=device)
|
||||
kv_lens = torch.tensor([45, 29], dtype=torch.int32, device=device)
|
||||
sm_scale = QK_NOPE_HEAD_DIM**-0.5
|
||||
|
||||
def run(
|
||||
ckv_cache: torch.Tensor,
|
||||
kpe_cache: torch.Tensor,
|
||||
**kwargs,
|
||||
) -> torch.Tensor:
|
||||
workspace = torch.empty(
|
||||
FLASHINFER_WORKSPACE_BUFFER_SIZE,
|
||||
dtype=torch.uint8,
|
||||
device=device,
|
||||
)
|
||||
wrapper = BatchMLAPagedAttentionWrapper(workspace, backend="fa3")
|
||||
wrapper.plan(
|
||||
qo_indptr,
|
||||
kv_indptr,
|
||||
kv_indices,
|
||||
kv_lens,
|
||||
num_heads,
|
||||
KV_LORA_RANK,
|
||||
0,
|
||||
page_size,
|
||||
False,
|
||||
sm_scale,
|
||||
q_data_type=torch.bfloat16,
|
||||
kv_data_type=ckv_cache.dtype,
|
||||
)
|
||||
return wrapper.run(q_nope, q_pe, ckv_cache, kpe_cache, **kwargs)
|
||||
|
||||
out_ref = run(ckv_ref, kpe_ref)
|
||||
out = run(ckv_fp8, kpe_fp8, ckv_scale=ckv_scale, kpe_scale=1.0)
|
||||
torch.testing.assert_close(out, out_ref, atol=2e-2, rtol=2e-2)
|
||||
|
||||
|
||||
@requires_sm10x
|
||||
def test_flashinfer_mla_decode_workspace_supports_autotune():
|
||||
"""VLLM's FlashInfer MLA decode workspace must be int8 for autotuning.
|
||||
|
||||
Model Runner V2's warmup autotunes ``trtllm_batch_decode_mla``, which makes
|
||||
the FlashInfer autotuner enumerate the CuteDSL tactic. That tactic asserts
|
||||
``workspace_buffer.dtype == torch.int8``; the trtllm-gen path (used for
|
||||
normal, non-autotuned inference) instead views the buffer as uint8, so a
|
||||
uint8 workspace only fails once the autotuner tries CuteDSL. That regressed
|
||||
every DeepSeek MLA test on Blackwell under V2 with
|
||||
``workspace_buffer must be torch.int8`` (vllm-project/vllm#46646).
|
||||
"""
|
||||
from flashinfer.autotuner import autotune
|
||||
|
||||
from vllm.v1.attention.backends.mla.flashinfer_mla import _get_workspace_buffer
|
||||
|
||||
torch.set_default_device("cuda")
|
||||
torch.manual_seed(0)
|
||||
|
||||
workspace_buffer = _get_workspace_buffer(return_lse=False)
|
||||
assert workspace_buffer.dtype == torch.int8
|
||||
|
||||
q, kv_cache, block_tables, seq_lens_tensor, max_seq_len = _make_decode_inputs(
|
||||
bs=1, block_size=64, dtype=torch.bfloat16
|
||||
)
|
||||
|
||||
# Under the autotuner the CuteDSL tactic is instantiated with our workspace;
|
||||
# a uint8 buffer raises AssertionError here, an int8 buffer succeeds.
|
||||
with torch.inference_mode(), autotune(True):
|
||||
trtllm_batch_decode_with_kv_cache_mla(
|
||||
query=q.unsqueeze(1),
|
||||
kv_cache=kv_cache.unsqueeze(1),
|
||||
workspace_buffer=workspace_buffer,
|
||||
qk_nope_head_dim=QK_NOPE_HEAD_DIM,
|
||||
kv_lora_rank=KV_LORA_RANK,
|
||||
qk_rope_head_dim=QK_ROPE_HEAD_DIM,
|
||||
block_tables=block_tables,
|
||||
seq_lens=seq_lens_tensor,
|
||||
max_seq_len=max_seq_len,
|
||||
bmm1_scale=SCALE,
|
||||
)
|
||||
@@ -1,842 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sm120", [False, True])
|
||||
def test_deepseek_v4_c128a_adaptive_width_has_capture_stable_stride(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
sm120: bool,
|
||||
):
|
||||
from vllm.models.deepseek_v4 import sparse_mla
|
||||
from vllm.platforms.interface import DeviceCapability
|
||||
|
||||
monkeypatch.setattr(
|
||||
sparse_mla.current_platform,
|
||||
"get_device_capability",
|
||||
lambda: DeviceCapability(12, 0) if sm120 else DeviceCapability(10, 0),
|
||||
)
|
||||
|
||||
device = torch.device("cuda")
|
||||
capacity_width = 512
|
||||
global_decode_buffer = torch.empty(
|
||||
(2, capacity_width), dtype=torch.int32, device=device
|
||||
)
|
||||
prefill_buffer = torch.empty_like(global_decode_buffer)
|
||||
kwargs = dict(
|
||||
positions=torch.tensor([255, 511, 383, 639], device=device),
|
||||
compress_ratio=128,
|
||||
num_decode_tokens=2,
|
||||
token_to_req_indices=torch.tensor(
|
||||
[0, 1, 0, 1], dtype=torch.int32, device=device
|
||||
),
|
||||
block_table=torch.tensor([[3], [5]], dtype=torch.int32, device=device),
|
||||
block_size=capacity_width,
|
||||
slot_mapping=torch.arange(4, dtype=torch.int64, device=device),
|
||||
global_decode_buffer=global_decode_buffer,
|
||||
decode_lens_buffer=torch.empty(2, dtype=torch.int32, device=device),
|
||||
prefill_buffer=prefill_buffer,
|
||||
)
|
||||
captured_decode, _, captured_prefill = sparse_mla.build_c128a_topk_metadata(
|
||||
max_compressed_tokens=256,
|
||||
**kwargs,
|
||||
)
|
||||
# SM120 keeps the decode view contiguous across the full buffer width;
|
||||
# other backends get the active-width slice. The prefill view is always
|
||||
# narrowed.
|
||||
expected_decode_width = capacity_width if sm120 else 256
|
||||
assert captured_decode.shape == (2, expected_decode_width)
|
||||
assert captured_prefill.shape == (2, 256)
|
||||
assert captured_decode.stride(0) == captured_prefill.stride(0) == capacity_width
|
||||
assert captured_decode.is_contiguous() == sm120
|
||||
|
||||
captured_rows = torch.empty((4, 4), dtype=torch.int32, device=device)
|
||||
captured_rows[:2].copy_(captured_decode[:, :4])
|
||||
captured_rows[2:].copy_(captured_prefill[:, :4])
|
||||
torch.accelerator.synchronize()
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
captured_rows[:2].copy_(captured_decode[:, :4])
|
||||
captured_rows[2:].copy_(captured_prefill[:, :4])
|
||||
|
||||
global_decode_buffer.fill_(-99)
|
||||
prefill_buffer.fill_(-99)
|
||||
sparse_mla.build_c128a_topk_metadata(
|
||||
max_compressed_tokens=128,
|
||||
**kwargs,
|
||||
)
|
||||
graph.replay()
|
||||
|
||||
assert captured_rows.cpu().tolist() == [
|
||||
[1536, 1537, -1, -1],
|
||||
[2560, 2561, 2562, 2563],
|
||||
[0, 1, 2, -1],
|
||||
[0, 1, 2, 3],
|
||||
]
|
||||
assert torch.all(global_decode_buffer[:, 128:] == -99)
|
||||
assert torch.all(prefill_buffer[:, 128:] == -99)
|
||||
|
||||
|
||||
def test_sparse_flashmla_metadata_smoke():
|
||||
import vllm.v1.attention.ops.flashmla as fm
|
||||
|
||||
ok, reason = fm.is_flashmla_sparse_supported()
|
||||
if not ok:
|
||||
pytest.skip(reason)
|
||||
|
||||
device = torch.device("cuda")
|
||||
batch_size = 1
|
||||
seqlen_q = 1
|
||||
num_heads_q = 128
|
||||
num_heads_k = 1
|
||||
q_seq_per_hk = seqlen_q * num_heads_q // num_heads_k
|
||||
topk = 128
|
||||
|
||||
cache_seqlens = torch.zeros(batch_size, dtype=torch.int32, device=device)
|
||||
|
||||
tile_md, num_splits = fm.get_mla_metadata(
|
||||
cache_seqlens,
|
||||
q_seq_per_hk,
|
||||
num_heads_k,
|
||||
num_heads_q=num_heads_q,
|
||||
topk=topk,
|
||||
is_fp8_kvcache=True,
|
||||
)
|
||||
assert isinstance(tile_md, fm.FlashMLASchedMeta)
|
||||
assert tile_md.tile_scheduler_metadata is None
|
||||
assert tile_md.num_splits is None
|
||||
assert num_splits is None
|
||||
|
||||
|
||||
def test_sparse_flashmla_decode_matches_cache_writer_scales():
|
||||
import vllm.v1.attention.ops.flashmla as fm
|
||||
from vllm import _custom_ops as ops
|
||||
|
||||
ok, reason = fm.is_flashmla_sparse_supported()
|
||||
if not ok:
|
||||
pytest.skip(reason)
|
||||
|
||||
device = torch.device("cuda")
|
||||
batch_size = 1
|
||||
seqlen_q = 1
|
||||
num_heads_q = 64
|
||||
head_dim_k = 576
|
||||
head_dim_v = 512
|
||||
num_heads_k = 1
|
||||
page_block_size = 64
|
||||
bytes_per_token = 656
|
||||
topk = 128
|
||||
|
||||
# Metadata
|
||||
q_seq_per_hk = seqlen_q * num_heads_q // num_heads_k
|
||||
# q_heads_per_hk = num_heads_q // num_heads_k
|
||||
cache_seqlens = torch.ones(batch_size, dtype=torch.int32, device=device)
|
||||
tile_md, num_splits = fm.get_mla_metadata(
|
||||
cache_seqlens,
|
||||
q_seq_per_hk,
|
||||
num_heads_k,
|
||||
num_heads_q=num_heads_q,
|
||||
topk=topk,
|
||||
is_fp8_kvcache=True,
|
||||
)
|
||||
|
||||
# Inputs
|
||||
q = torch.zeros(
|
||||
(batch_size, seqlen_q, num_heads_q, head_dim_k),
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
cache_rows = torch.zeros(
|
||||
(1, page_block_size, bytes_per_token), dtype=torch.uint8, device=device
|
||||
)
|
||||
# 336 / 448 = 0.75 distinguishes arbitrary FP32 from power-of-two scales.
|
||||
kv_c = torch.full((1, 512), 336.0, dtype=torch.bfloat16, device=device)
|
||||
kv_c[:, -128:] = 0
|
||||
ops.concat_and_cache_mla(
|
||||
kv_c,
|
||||
torch.zeros((1, 64), dtype=torch.bfloat16, device=device),
|
||||
cache_rows,
|
||||
torch.zeros(1, dtype=torch.int64, device=device),
|
||||
"fp8_ds_mla",
|
||||
torch.ones(1, dtype=torch.float32, device=device),
|
||||
)
|
||||
scales = cache_rows[0, 0].view(torch.float32)[128:132]
|
||||
expected_scales = torch.tensor([1.0, 1.0, 1.0, 2**-13], device=device)
|
||||
torch.testing.assert_close(scales, expected_scales, rtol=0, atol=0)
|
||||
cached_nope = cache_rows[0, 0, :512].view(torch.float8_e4m3fn).float()
|
||||
expected = (cached_nope * scales.repeat_interleave(128)).to(torch.bfloat16)
|
||||
|
||||
k_cache = cache_rows.unsqueeze(2)
|
||||
indices = torch.full(
|
||||
(batch_size, seqlen_q, topk), -1, dtype=torch.int32, device=device
|
||||
)
|
||||
indices[..., 0] = 0
|
||||
# SM90 sparse decode only supports a fixed top-k width. Padding with -1
|
||||
# exercises the same cache data on SM90 and SM100 without topk_length.
|
||||
|
||||
block_table = torch.zeros((batch_size, 128), dtype=torch.int32, device=device)
|
||||
out, lse = fm.flash_mla_with_kvcache(
|
||||
q,
|
||||
k_cache,
|
||||
block_table,
|
||||
cache_seqlens,
|
||||
head_dim_v,
|
||||
tile_md,
|
||||
num_splits,
|
||||
indices=indices,
|
||||
is_fp8_kvcache=True,
|
||||
)
|
||||
assert out.shape[0] == batch_size
|
||||
assert out.shape[-1] == head_dim_v
|
||||
assert lse.shape[0] == batch_size
|
||||
torch.testing.assert_close(
|
||||
out, expected.view(1, 1, 1, 512).expand_as(out), rtol=0, atol=0
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("h_q", [64, 128])
|
||||
def test_sparse_flashmla_prefill_smoke(h_q: int):
|
||||
import vllm.v1.attention.ops.flashmla as fm
|
||||
|
||||
ok, reason = fm.is_flashmla_sparse_supported()
|
||||
if not ok:
|
||||
pytest.skip(reason)
|
||||
|
||||
device = torch.device("cuda")
|
||||
torch.manual_seed(0)
|
||||
s_q = 1
|
||||
s_kv = 8
|
||||
h_kv = 1
|
||||
d_qk = 576
|
||||
d_v = 512
|
||||
topk = 128
|
||||
q = torch.randn((s_q, h_q, d_qk), dtype=torch.bfloat16, device=device)
|
||||
kv = torch.randn((s_kv, h_kv, d_qk), dtype=torch.bfloat16, device=device)
|
||||
indices = torch.randint(s_kv, (s_q, h_kv, topk), dtype=torch.int32, device=device)
|
||||
reference_indices = indices.clone()
|
||||
reference_indices[..., 1:] = -1
|
||||
kwargs = {"topk_length": torch.ones(1, dtype=torch.int32, device=device)}
|
||||
reference = fm.flash_mla_sparse_fwd(q, kv, reference_indices, 1.0, d_v, **kwargs)
|
||||
actual = fm.flash_mla_sparse_fwd(q, kv, indices, 1.0, d_v, **kwargs)
|
||||
|
||||
for actual_tensor, reference_tensor in zip(actual, reference):
|
||||
torch.testing.assert_close(actual_tensor, reference_tensor, rtol=0, atol=0)
|
||||
assert actual[0].shape == (s_q, h_q, d_v)
|
||||
|
||||
|
||||
def test_deepseek_v4_prefill_chunk_planning_expands_for_short_sequences():
|
||||
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
|
||||
|
||||
metadata = DeepseekSparseSWAMetadata(
|
||||
block_table=torch.empty(0, dtype=torch.int32),
|
||||
slot_mapping=torch.empty(0, dtype=torch.int32),
|
||||
block_size=64,
|
||||
num_prefills=5,
|
||||
prefill_seq_lens_cpu=torch.tensor([80, 96, 112, 128, 144], dtype=torch.int32),
|
||||
prefill_query_lens_cpu=torch.tensor([4, 4, 4, 4, 4], dtype=torch.int32),
|
||||
prefill_window_size=64,
|
||||
prefill_max_model_len=1024,
|
||||
prefill_max_num_batched_tokens=128,
|
||||
)
|
||||
|
||||
chunk_plan = metadata.get_prefill_chunk_plan(compress_ratio=4, prefill_chunk_size=4)
|
||||
|
||||
# the adaptive plan keeps all 5 in one chunk
|
||||
assert chunk_plan == [(0, 5, 36, 103)]
|
||||
|
||||
|
||||
def test_flashinfer_sparse_indices_cache(monkeypatch):
|
||||
from vllm.models.deepseek_v4.nvidia import flashinfer_sparse as flashinfer_mod
|
||||
from vllm.models.deepseek_v4.sparse_mla import DeepseekV4FlashMLAMetadata
|
||||
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
|
||||
|
||||
builder_calls = 0
|
||||
|
||||
def fake_build(*args, **kwargs):
|
||||
nonlocal builder_calls
|
||||
builder_calls += 1
|
||||
return (
|
||||
torch.tensor([[builder_calls]], dtype=torch.int32),
|
||||
torch.tensor([builder_calls], dtype=torch.int32),
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
flashinfer_mod, "build_flashinfer_mixed_sparse_indices", fake_build
|
||||
)
|
||||
|
||||
def make_attn(compress_ratio: int, topk_width: int):
|
||||
attn = object.__new__(flashinfer_mod.DeepseekV4FlashInferMLAAttention)
|
||||
attn.compress_ratio = compress_ratio
|
||||
attn.window_size = 4
|
||||
attn.topk_indices_buffer = torch.tensor(
|
||||
[[0, 1], [2, 3], [4, 5]], dtype=torch.int32
|
||||
)[:, :topk_width]
|
||||
return attn
|
||||
|
||||
def make_swa_metadata():
|
||||
return DeepseekSparseSWAMetadata(
|
||||
block_table=torch.tensor([[0, 1], [2, 3]], dtype=torch.int32),
|
||||
slot_mapping=torch.tensor([0, 1], dtype=torch.int64),
|
||||
block_size=64,
|
||||
seq_lens=torch.tensor([8, 10], dtype=torch.int32),
|
||||
query_start_loc=torch.tensor([0, 1, 3], dtype=torch.int32),
|
||||
query_start_loc_cpu=torch.tensor([0, 1, 3], dtype=torch.int32),
|
||||
token_to_req_indices=torch.tensor([0, 1, 1], dtype=torch.int32),
|
||||
decode_swa_indices=torch.tensor([[5, 6, -1, -1]], dtype=torch.int32),
|
||||
decode_swa_lens=torch.tensor([2], dtype=torch.int32),
|
||||
decode_swa_width=4,
|
||||
is_valid_token=torch.tensor([True], dtype=torch.bool),
|
||||
num_decodes=1,
|
||||
num_prefills=1,
|
||||
num_decode_tokens=1,
|
||||
num_prefill_tokens=2,
|
||||
)
|
||||
|
||||
def make_flashmla_metadata():
|
||||
return DeepseekV4FlashMLAMetadata(
|
||||
num_reqs=2,
|
||||
max_query_len=2,
|
||||
max_seq_len=10,
|
||||
num_actual_tokens=3,
|
||||
query_start_loc=torch.tensor([0, 1, 3], dtype=torch.int32),
|
||||
slot_mapping=torch.tensor([0, 1, 2], dtype=torch.int64),
|
||||
block_table=torch.tensor([[0, 1], [2, 3]], dtype=torch.int32),
|
||||
req_id_per_token=torch.tensor([0, 1, 1], dtype=torch.int32),
|
||||
block_size=256,
|
||||
topk_tokens=2,
|
||||
c128a_global_decode_topk_indices=torch.tensor(
|
||||
[[[9, 10]]], dtype=torch.int32
|
||||
),
|
||||
c128a_decode_topk_lens=torch.tensor([2], dtype=torch.int32),
|
||||
c128a_prefill_topk_indices=torch.tensor(
|
||||
[[0, 1], [1, 2]], dtype=torch.int32
|
||||
),
|
||||
)
|
||||
|
||||
swa_attn = make_attn(1, 0)
|
||||
swa_metadata = make_swa_metadata()
|
||||
_, _, sparse_indices_first, sparse_lens_first = (
|
||||
swa_attn._build_sparse_index_metadata(
|
||||
kv_cache=None,
|
||||
swa_k_cache=torch.empty((1, 64, 512), dtype=torch.bfloat16),
|
||||
swa_metadata=swa_metadata,
|
||||
attn_metadata=None,
|
||||
swa_only=True,
|
||||
)
|
||||
)
|
||||
_, _, sparse_indices_second, sparse_lens_second = (
|
||||
swa_attn._build_sparse_index_metadata(
|
||||
kv_cache=None,
|
||||
swa_k_cache=torch.empty((1, 64, 512), dtype=torch.bfloat16),
|
||||
swa_metadata=swa_metadata,
|
||||
attn_metadata=None,
|
||||
swa_only=True,
|
||||
)
|
||||
)
|
||||
assert builder_calls == 1
|
||||
assert sparse_indices_first is sparse_indices_second
|
||||
assert sparse_lens_first is sparse_lens_second
|
||||
|
||||
c128a_attn = make_attn(128, 2)
|
||||
c128a_metadata = make_swa_metadata()
|
||||
c128a_flashmla_md = make_flashmla_metadata()
|
||||
_, _, sparse_indices_first, sparse_lens_first = (
|
||||
c128a_attn._build_sparse_index_metadata(
|
||||
kv_cache=torch.empty((1, 2, 512), dtype=torch.bfloat16),
|
||||
swa_k_cache=torch.empty((1, 64, 512), dtype=torch.bfloat16),
|
||||
swa_metadata=c128a_metadata,
|
||||
attn_metadata=c128a_flashmla_md,
|
||||
swa_only=False,
|
||||
)
|
||||
)
|
||||
_, _, sparse_indices_second, sparse_lens_second = (
|
||||
c128a_attn._build_sparse_index_metadata(
|
||||
kv_cache=torch.empty((1, 2, 512), dtype=torch.bfloat16),
|
||||
swa_k_cache=torch.empty((1, 64, 512), dtype=torch.bfloat16),
|
||||
swa_metadata=c128a_metadata,
|
||||
attn_metadata=c128a_flashmla_md,
|
||||
swa_only=False,
|
||||
)
|
||||
)
|
||||
|
||||
assert builder_calls == 2
|
||||
assert sparse_indices_first is sparse_indices_second
|
||||
assert sparse_lens_first is sparse_lens_second
|
||||
|
||||
c4a_attn = make_attn(4, 2)
|
||||
c4a_metadata = make_swa_metadata()
|
||||
c4a_flashmla_md = make_flashmla_metadata()
|
||||
c4a_flashmla_md.c128a_global_decode_topk_indices = None
|
||||
c4a_flashmla_md.c128a_decode_topk_lens = None
|
||||
c4a_flashmla_md.c128a_prefill_topk_indices = None
|
||||
_, _, sparse_indices_third, sparse_lens_third = (
|
||||
c4a_attn._build_sparse_index_metadata(
|
||||
kv_cache=torch.empty((1, 2, 512), dtype=torch.bfloat16),
|
||||
swa_k_cache=torch.empty((1, 64, 512), dtype=torch.bfloat16),
|
||||
swa_metadata=c4a_metadata,
|
||||
attn_metadata=c4a_flashmla_md,
|
||||
swa_only=False,
|
||||
)
|
||||
)
|
||||
_, _, sparse_indices_fourth, sparse_lens_fourth = (
|
||||
c4a_attn._build_sparse_index_metadata(
|
||||
kv_cache=torch.empty((1, 2, 512), dtype=torch.bfloat16),
|
||||
swa_k_cache=torch.empty((1, 64, 512), dtype=torch.bfloat16),
|
||||
swa_metadata=c4a_metadata,
|
||||
attn_metadata=c4a_flashmla_md,
|
||||
swa_only=False,
|
||||
)
|
||||
)
|
||||
|
||||
assert builder_calls == 4
|
||||
assert sparse_indices_third is not sparse_indices_fourth
|
||||
assert sparse_lens_third is not sparse_lens_fourth
|
||||
|
||||
|
||||
def test_flashinfer_sparse_index_preserves_logical_window(monkeypatch):
|
||||
from vllm.models.deepseek_v4.nvidia import flashinfer_sparse as flashinfer_mod
|
||||
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
|
||||
|
||||
captured_shapes_and_windows: list[tuple[int, int]] = []
|
||||
|
||||
def fake_build(*args, **kwargs):
|
||||
# window_size is the 12th positional arg of
|
||||
# build_flashinfer_mixed_sparse_indices.
|
||||
captured_shapes_and_windows.append((args[0].shape[-1], args[11]))
|
||||
num_tokens = args[0].shape[0] + args[3].shape[0]
|
||||
return (
|
||||
torch.zeros((num_tokens, 1), dtype=torch.int32),
|
||||
torch.zeros((num_tokens,), dtype=torch.int32),
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
flashinfer_mod, "build_flashinfer_mixed_sparse_indices", fake_build
|
||||
)
|
||||
|
||||
attn = object.__new__(flashinfer_mod.DeepseekV4FlashInferMLAAttention)
|
||||
attn.compress_ratio = 1
|
||||
attn.window_size = 4
|
||||
attn.topk_indices_buffer = torch.zeros((4, 0), dtype=torch.int32)
|
||||
|
||||
wide_width = 8
|
||||
wide_indices = torch.full((1, wide_width), -1, dtype=torch.int32)
|
||||
wide_indices[0, :2] = torch.tensor([5, 6], dtype=torch.int32)
|
||||
wide_metadata = DeepseekSparseSWAMetadata(
|
||||
block_table=torch.tensor([[0, 1]], dtype=torch.int32),
|
||||
slot_mapping=torch.tensor([0], dtype=torch.int64),
|
||||
block_size=64,
|
||||
seq_lens=torch.tensor([8], dtype=torch.int32),
|
||||
query_start_loc=torch.tensor([0, 1], dtype=torch.int32),
|
||||
query_start_loc_cpu=torch.tensor([0, 1], dtype=torch.int32),
|
||||
token_to_req_indices=torch.tensor([0], dtype=torch.int32),
|
||||
decode_swa_indices=wide_indices,
|
||||
decode_swa_lens=torch.tensor([2], dtype=torch.int32),
|
||||
decode_swa_width=wide_width,
|
||||
is_valid_token=torch.tensor([True], dtype=torch.bool),
|
||||
num_decodes=1,
|
||||
num_prefills=0,
|
||||
num_decode_tokens=1,
|
||||
num_prefill_tokens=0,
|
||||
)
|
||||
attn._build_sparse_index_metadata(
|
||||
kv_cache=None,
|
||||
swa_k_cache=torch.empty((1, 64, 512), dtype=torch.bfloat16),
|
||||
swa_metadata=wide_metadata,
|
||||
attn_metadata=None,
|
||||
swa_only=True,
|
||||
)
|
||||
assert captured_shapes_and_windows == [(wide_width, attn.window_size)]
|
||||
|
||||
empty_width = 8
|
||||
empty_metadata = DeepseekSparseSWAMetadata(
|
||||
block_table=torch.tensor([[0, 1]], dtype=torch.int32),
|
||||
slot_mapping=torch.tensor([0, 1], dtype=torch.int64),
|
||||
block_size=64,
|
||||
seq_lens=torch.tensor([8], dtype=torch.int32),
|
||||
query_start_loc=torch.tensor([0, 2], dtype=torch.int32),
|
||||
query_start_loc_cpu=torch.tensor([0, 2], dtype=torch.int32),
|
||||
token_to_req_indices=torch.tensor([0, 0], dtype=torch.int32),
|
||||
decode_swa_indices=torch.empty((0, 1, empty_width), dtype=torch.int32),
|
||||
decode_swa_lens=torch.empty((0,), dtype=torch.int32),
|
||||
decode_swa_width=empty_width,
|
||||
is_valid_token=torch.tensor([True, True], dtype=torch.bool),
|
||||
num_decodes=0,
|
||||
num_prefills=1,
|
||||
num_decode_tokens=0,
|
||||
num_prefill_tokens=2,
|
||||
)
|
||||
attn._build_sparse_index_metadata(
|
||||
kv_cache=None,
|
||||
swa_k_cache=torch.empty((1, 64, 512), dtype=torch.bfloat16),
|
||||
swa_metadata=empty_metadata,
|
||||
attn_metadata=None,
|
||||
swa_only=True,
|
||||
)
|
||||
assert captured_shapes_and_windows == [
|
||||
(wide_width, attn.window_size),
|
||||
(empty_width, attn.window_size),
|
||||
]
|
||||
|
||||
|
||||
def test_flashinfer_mixed_sparse_indices_separates_window_and_padded_width():
|
||||
from vllm.models.deepseek_v4.common.ops.cache_utils import (
|
||||
build_flashinfer_mixed_sparse_indices,
|
||||
)
|
||||
|
||||
device = torch.device("cuda")
|
||||
padded_width = 8
|
||||
logical_window = 4
|
||||
sparse_indices, sparse_lens = build_flashinfer_mixed_sparse_indices(
|
||||
decode_swa_indices=torch.empty(
|
||||
(0, padded_width), dtype=torch.int32, device=device
|
||||
),
|
||||
decode_compressed_indices=None,
|
||||
decode_compressed_topk_lens=None,
|
||||
prefill_topk_indices=torch.empty((1, 0), dtype=torch.int32, device=device),
|
||||
query_start_loc=torch.tensor([0, 1], dtype=torch.int32, device=device),
|
||||
seq_lens=torch.tensor([logical_window], dtype=torch.int32, device=device),
|
||||
token_to_req_indices=torch.tensor([0], dtype=torch.int32, device=device),
|
||||
swa_block_table=torch.tensor([[0]], dtype=torch.int32, device=device),
|
||||
swa_block_size=64,
|
||||
compressed_block_table=None,
|
||||
compressed_block_size=64,
|
||||
window_size=logical_window,
|
||||
compress_ratio=1,
|
||||
topk=0,
|
||||
)
|
||||
|
||||
assert sparse_indices.shape == (1, padded_width)
|
||||
assert sparse_indices[0].cpu().tolist() == [0, 1, 2, 3, -1, -1, -1, -1]
|
||||
assert sparse_lens.cpu().tolist() == [padded_width]
|
||||
|
||||
|
||||
def test_flashinfer_mixed_sparse_indices_noncausal_rows_have_no_active_gaps():
|
||||
"""A DSpark non-causal row keeps -1 out of the kernel's active ranges.
|
||||
|
||||
Each draft token's causal window holds its first visible entries, the rest
|
||||
of the block starts at column `window`, and the compressed entries follow
|
||||
directly; `sparse_topk_lens` ends right after them.
|
||||
"""
|
||||
from vllm.models.deepseek_v4.common.ops.cache_utils import (
|
||||
build_flashinfer_mixed_sparse_indices,
|
||||
)
|
||||
|
||||
device = torch.device("cuda")
|
||||
window, width = 4, 8
|
||||
# One request: 1 context token, a 2-token draft block, slots 0..2 visible.
|
||||
decode_swa = torch.full((2, width), -1, dtype=torch.int32, device=device)
|
||||
decode_swa[:, :3] = torch.arange(3, dtype=torch.int32, device=device)
|
||||
compressed = torch.tensor(
|
||||
[[100, 101, -1, -1]] * 2, dtype=torch.int32, device=device
|
||||
)
|
||||
sparse_indices, sparse_lens = build_flashinfer_mixed_sparse_indices(
|
||||
decode_swa_indices=decode_swa,
|
||||
decode_compressed_indices=compressed,
|
||||
decode_compressed_topk_lens=torch.tensor(
|
||||
[2, 2], dtype=torch.int32, device=device
|
||||
),
|
||||
prefill_topk_indices=torch.empty((0, 4), dtype=torch.int32, device=device),
|
||||
query_start_loc=torch.tensor([0, 2], dtype=torch.int32, device=device),
|
||||
seq_lens=torch.tensor([3], dtype=torch.int32, device=device),
|
||||
token_to_req_indices=torch.tensor([0, 0], dtype=torch.int32, device=device),
|
||||
swa_block_table=torch.tensor([[0]], dtype=torch.int32, device=device),
|
||||
swa_block_size=64,
|
||||
compressed_block_table=torch.tensor([[0]], dtype=torch.int32, device=device),
|
||||
compressed_block_size=64,
|
||||
window_size=window,
|
||||
compress_ratio=128,
|
||||
topk=0,
|
||||
)
|
||||
|
||||
assert sparse_indices.cpu().tolist() == [
|
||||
# pos 1: window [0, 1]; slot 2 spills to column 4; compressed after it.
|
||||
[0, 1, -1, -1, 2, 100, 101, -1, -1, -1, -1, -1],
|
||||
# pos 2: window [0, 1, 2]; nothing spills; compressed at column 4.
|
||||
[0, 1, 2, -1, 100, 101, -1, -1, -1, -1, -1, -1],
|
||||
]
|
||||
assert sparse_lens.cpu().tolist() == [7, 6]
|
||||
|
||||
|
||||
def test_flashinfer_mixed_sparse_indices_noncausal_pad_rows_keep_min_length():
|
||||
"""A CUDA-graph pad token of the DSpark graph (a zero-length request) gets
|
||||
a causal pad row's shape instead of negative spill arithmetic."""
|
||||
from vllm.models.deepseek_v4.common.ops.cache_utils import (
|
||||
build_flashinfer_mixed_sparse_indices,
|
||||
)
|
||||
|
||||
device = torch.device("cuda")
|
||||
window, width = 128, 192
|
||||
decode_swa = torch.full((3, width), -1, dtype=torch.int32, device=device)
|
||||
decode_swa[:2, :3] = torch.arange(3, dtype=torch.int32, device=device)
|
||||
sparse_indices, sparse_lens = build_flashinfer_mixed_sparse_indices(
|
||||
decode_swa_indices=decode_swa,
|
||||
decode_compressed_indices=None,
|
||||
decode_compressed_topk_lens=None,
|
||||
prefill_topk_indices=torch.empty((0, 0), dtype=torch.int32, device=device),
|
||||
# Request 1 is the padding: no query tokens, seq_len 0.
|
||||
query_start_loc=torch.tensor([0, 2, 2], dtype=torch.int32, device=device),
|
||||
seq_lens=torch.tensor([3, 0], dtype=torch.int32, device=device),
|
||||
token_to_req_indices=torch.tensor([0, 0, 1], dtype=torch.int32, device=device),
|
||||
swa_block_table=torch.zeros((2, 1), dtype=torch.int32, device=device),
|
||||
swa_block_size=64,
|
||||
compressed_block_table=None,
|
||||
compressed_block_size=64,
|
||||
window_size=window,
|
||||
compress_ratio=1,
|
||||
topk=0,
|
||||
)
|
||||
|
||||
assert sparse_lens.cpu().tolist() == [129, 128, 128]
|
||||
assert (sparse_indices[2] == -1).all()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["deepseek_v4", "deepseek_v41"])
|
||||
@pytest.mark.parametrize("page_padding", [0, 64])
|
||||
def test_flashinfer_sparse_forward_reads_packed_kv_rows(model, page_padding):
|
||||
"""Decode and prefill must read storage rows after vLLM remaps packed pages."""
|
||||
import importlib
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
|
||||
|
||||
if not current_platform.is_device_capability_family(100):
|
||||
pytest.skip("Requires FlashInfer TRTLLM sparse MLA on SM100/SM103")
|
||||
mod = importlib.import_module(f"vllm.models.{model}.nvidia.flashinfer_sparse")
|
||||
device = "cuda"
|
||||
block_size = 64
|
||||
storage = torch.full(
|
||||
(3, block_size + page_padding, 512), 99, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
cache = storage[:, :block_size]
|
||||
cache[1].fill_(2)
|
||||
cache[2].fill_(3)
|
||||
attn = object.__new__(mod.DeepseekV4FlashInferMLAAttention)
|
||||
attn.compress_ratio = 1
|
||||
attn.window_size = 128
|
||||
attn.scale = 512**-0.5
|
||||
attn.kv_cache_torch_dtype = torch.bfloat16
|
||||
attn.attn_sink = torch.full((64,), -float("inf"), device=device)
|
||||
attn.topk_indices_buffer = torch.empty((3, 0), dtype=torch.int32, device=device)
|
||||
decode_indices = torch.full((1, 128), -1, dtype=torch.int32, device=device)
|
||||
decode_indices[0, :4] = torch.arange(64, 68, dtype=torch.int32, device=device)
|
||||
query_start = torch.tensor([0, 1, 3], dtype=torch.int32)
|
||||
metadata = DeepseekSparseSWAMetadata(
|
||||
block_table=torch.tensor([[1], [2]], dtype=torch.int32, device=device),
|
||||
slot_mapping=torch.tensor([67, 130, 131], device=device),
|
||||
block_size=block_size,
|
||||
seq_lens=torch.tensor([4, 4], dtype=torch.int32, device=device),
|
||||
query_start_loc=query_start.to(device),
|
||||
query_start_loc_cpu=query_start,
|
||||
token_to_req_indices=torch.tensor([0, 1, 1], dtype=torch.int32, device=device),
|
||||
decode_swa_indices=decode_indices,
|
||||
decode_swa_lens=torch.tensor([4], dtype=torch.int32, device=device),
|
||||
decode_swa_width=128,
|
||||
is_valid_token=torch.ones(3, dtype=torch.bool, device=device),
|
||||
replay_start=torch.zeros(2, dtype=torch.int32, device=device),
|
||||
num_decodes=1,
|
||||
num_prefills=1,
|
||||
num_decode_tokens=1,
|
||||
num_prefill_tokens=2,
|
||||
max_decode_query_len=1,
|
||||
)
|
||||
query = torch.zeros((3, 64, 512), dtype=torch.bfloat16, device=device)
|
||||
output = torch.empty_like(query)
|
||||
attn._forward(query, None, cache, metadata, None, True, output)
|
||||
expected = torch.tensor([2, 3, 3], dtype=output.dtype, device=device)
|
||||
torch.testing.assert_close(output, expected[:, None, None].expand_as(output))
|
||||
|
||||
|
||||
def _make_rope_quant_attn(mod, num_rows: int, device: str):
|
||||
"""A bare DSv4 FlashInfer layer set up for RopeQuant: 128 heads, fp8 KV."""
|
||||
from types import SimpleNamespace
|
||||
|
||||
heads, groups, rank = 128, 16, 256
|
||||
angles = torch.arange(2048, device=device, dtype=torch.float32)[:, None] * 0.05
|
||||
angles = angles + torch.arange(32, device=device)[None] * 0.1
|
||||
attn = object.__new__(mod.DeepseekV4FlashInferMLAAttention)
|
||||
attn.compress_ratio = 1
|
||||
attn.window_size = 128
|
||||
attn.scale = 512**-0.5
|
||||
attn.kv_cache_torch_dtype = torch.float8_e4m3fn
|
||||
attn._flashinfer_fp8_bmm1_scale = attn.scale
|
||||
attn._flashinfer_fp8_bmm2_scale = 1.0
|
||||
attn.attn_sink = torch.linspace(-1, 1, heads, device=device)
|
||||
attn.padded_heads = heads
|
||||
attn.topk_indices_buffer = torch.empty(
|
||||
(num_rows, 0), dtype=torch.int32, device=device
|
||||
)
|
||||
attn.rotary_emb = SimpleNamespace(
|
||||
cos_sin_cache=torch.cat((angles.cos(), angles.sin()), -1).contiguous()
|
||||
)
|
||||
attn.n_local_heads, attn.n_local_groups, attn.o_lora_rank = heads, groups, rank
|
||||
attn.nope_head_dim, attn.rope_head_dim = 448, 64
|
||||
attn._einsum_recipe, attn._tma_aligned_scales = (1, 1, 128), True
|
||||
attn.wo_a = SimpleNamespace(
|
||||
weight=(torch.randn((groups, rank, 4096), device=device) * 0.05).to(
|
||||
torch.float8_e4m3fn
|
||||
),
|
||||
weight_scale=torch.ones((groups, rank, 32), device=device),
|
||||
)
|
||||
attn.wo_b = lambda z: z
|
||||
return attn
|
||||
|
||||
|
||||
def _rope_quant_vs_unfused(attn, q, cache, metadata, positions, num_tokens):
|
||||
"""Run `_forward` unfused and fused; return (bf16 attention, z_unfused, z_fused)."""
|
||||
from vllm.models.deepseek_v4.nvidia.ops.o_proj import rope_quant_attn_out
|
||||
|
||||
heads, groups = attn.n_local_heads, attn.n_local_groups
|
||||
unfused = torch.empty(q.shape, dtype=torch.bfloat16, device=q.device)
|
||||
attn._forward(q, None, cache, metadata, None, True, unfused)
|
||||
fused = rope_quant_attn_out(q.shape[0], q.device)
|
||||
metadata.flashinfer_sparse_index_cache.clear()
|
||||
attn._forward(q, None, cache, metadata, None, True, fused)
|
||||
z_unfused = attn._o_proj(unfused[:num_tokens, :heads], positions)
|
||||
z_fused = attn._o_proj(fused, positions)[:num_tokens]
|
||||
return unfused[:num_tokens], z_unfused, z_fused.view(num_tokens, groups, -1)
|
||||
|
||||
|
||||
def _assert_projects_alike(z_unfused, z_fused):
|
||||
z_unfused = z_unfused.float().view_as(z_fused)
|
||||
rel = (z_fused.float() - z_unfused).norm() / z_unfused.norm()
|
||||
assert rel < 2e-2, rel
|
||||
|
||||
|
||||
def _skip_unless_rope_quant():
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_device_capability_family(100):
|
||||
pytest.skip("Requires FlashInfer TRTLLM sparse MLA on SM100/SM103")
|
||||
|
||||
|
||||
def test_flashinfer_rope_quant_matches_unfused_o_proj():
|
||||
"""RopeQuant's fused output must project like the bf16 + inv-RoPE/quant path.
|
||||
|
||||
One mixed step (two decodes, a three-token prefill) plus CUDA-graph pad rows,
|
||||
so both calls run over the full buffer with un-rebased cum_seq_lens_q.
|
||||
"""
|
||||
from vllm.models.deepseek_v4.nvidia import flashinfer_sparse as mod
|
||||
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
|
||||
|
||||
_skip_unless_rope_quant()
|
||||
torch.manual_seed(0)
|
||||
device = "cuda"
|
||||
block_size = 64
|
||||
q_lens, seq_lens = [1, 1, 3], [5, 9, 70]
|
||||
num_tokens, num_rows = sum(q_lens), sum(q_lens) + 3
|
||||
cache = torch.randn((3, block_size, 512), device=device).to(torch.float8_e4m3fn)
|
||||
attn = _make_rope_quant_attn(mod, num_rows, device)
|
||||
|
||||
decode_indices = torch.full((2, 128), -1, dtype=torch.int32, device=device)
|
||||
for i, seq_len in enumerate(seq_lens[:2]):
|
||||
decode_indices[i, :seq_len] = torch.arange(
|
||||
i * block_size, i * block_size + seq_len, device=device
|
||||
)
|
||||
query_start = torch.tensor([0, 1, 2, 5], dtype=torch.int32)
|
||||
metadata = DeepseekSparseSWAMetadata(
|
||||
block_table=torch.tensor([[0], [1], [2]], dtype=torch.int32, device=device),
|
||||
slot_mapping=torch.zeros(num_tokens, dtype=torch.int64, device=device),
|
||||
block_size=block_size,
|
||||
seq_lens=torch.tensor(seq_lens, dtype=torch.int32, device=device),
|
||||
query_start_loc=query_start.to(device),
|
||||
query_start_loc_cpu=query_start,
|
||||
token_to_req_indices=torch.tensor(
|
||||
[0, 1, 2, 2, 2], dtype=torch.int32, device=device
|
||||
),
|
||||
decode_swa_indices=decode_indices,
|
||||
decode_swa_lens=torch.tensor(seq_lens[:2], dtype=torch.int32, device=device),
|
||||
decode_swa_width=128,
|
||||
is_valid_token=torch.ones(num_tokens, dtype=torch.bool, device=device),
|
||||
num_decodes=2,
|
||||
num_prefills=1,
|
||||
num_decode_tokens=2,
|
||||
num_prefill_tokens=3,
|
||||
max_decode_query_len=1,
|
||||
)
|
||||
positions = torch.tensor([4, 8, 67, 68, 69], device=device)
|
||||
q = torch.randn((num_rows, 128, 512), device=device).to(torch.float8_e4m3fn)
|
||||
_, z_unfused, z_fused = _rope_quant_vs_unfused(
|
||||
attn, q, cache, metadata, positions, num_tokens
|
||||
)
|
||||
_assert_projects_alike(z_unfused, z_fused)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("context_len", [0, 20, 127, 900])
|
||||
def test_flashinfer_dspark_noncausal_block_sees_future_tokens(context_len):
|
||||
"""Every DSpark draft token attends to the whole block, fused or not.
|
||||
|
||||
The kernel counts every entry of its active index ranges as a key, so the
|
||||
builder must place the block around each query's causal window without -1
|
||||
gaps, down to an empty context.
|
||||
"""
|
||||
from vllm.models.deepseek_v4.nvidia import flashinfer_sparse as mod
|
||||
from vllm.v1.attention.backends.mla.compressor_utils import (
|
||||
get_dspark_swa_index_width,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
|
||||
|
||||
_skip_unless_rope_quant()
|
||||
torch.manual_seed(0)
|
||||
device = "cuda"
|
||||
block_size, window, draft = 64, 128, 4
|
||||
width = get_dspark_swa_index_width(window, draft)
|
||||
contexts = [context_len, context_len + 37]
|
||||
num_reqs, num_tokens = len(contexts), len(contexts) * draft
|
||||
num_rows = num_tokens + 2
|
||||
pages = -(-(max(contexts) + draft) // block_size)
|
||||
cache = torch.randn((num_reqs * pages, block_size, 512), device=device)
|
||||
cache = cache.clamp_(-1, 1).to(torch.float8_e4m3fn)
|
||||
attn = _make_rope_quant_attn(mod, num_rows, device)
|
||||
attn.attn_sink = torch.full((128,), -float("inf"), device=device)
|
||||
|
||||
# As the DSpark SWA builder lays them out: the trailing window of context
|
||||
# and the whole block, contiguous from column 0.
|
||||
decode_indices = torch.full((num_tokens, width), -1, dtype=torch.int32)
|
||||
visible, positions = [], []
|
||||
for r, ctx in enumerate(contexts):
|
||||
slots = torch.arange(max(ctx - window, 0), ctx + draft) + r * pages * block_size
|
||||
for k in range(draft):
|
||||
decode_indices[r * draft + k, : slots.numel()] = slots.to(torch.int32)
|
||||
visible.append(slots)
|
||||
positions.append(ctx + k)
|
||||
query_start = torch.arange(0, num_tokens + 1, draft, dtype=torch.int32)
|
||||
seq_lens = torch.tensor([c + draft for c in contexts], dtype=torch.int32)
|
||||
metadata = DeepseekSparseSWAMetadata(
|
||||
block_table=torch.arange(
|
||||
num_reqs * pages, dtype=torch.int32, device=device
|
||||
).view(num_reqs, pages),
|
||||
slot_mapping=torch.zeros(num_tokens, dtype=torch.int64, device=device),
|
||||
block_size=block_size,
|
||||
seq_lens=seq_lens.to(device),
|
||||
query_start_loc=query_start.to(device),
|
||||
query_start_loc_cpu=query_start,
|
||||
token_to_req_indices=torch.arange(
|
||||
num_reqs, dtype=torch.int32, device=device
|
||||
).repeat_interleave(draft),
|
||||
decode_swa_indices=decode_indices.to(device),
|
||||
decode_swa_lens=torch.tensor(
|
||||
[v.numel() for v in visible], dtype=torch.int32, device=device
|
||||
),
|
||||
decode_swa_width=width,
|
||||
is_valid_token=torch.ones(num_tokens, dtype=torch.bool, device=device),
|
||||
num_decodes=num_reqs,
|
||||
num_prefills=0,
|
||||
num_decode_tokens=num_tokens,
|
||||
num_prefill_tokens=0,
|
||||
max_decode_query_len=draft,
|
||||
)
|
||||
q = torch.randn((num_rows, 128, 512), device=device).clamp_(-1, 1)
|
||||
q = q.to(torch.float8_e4m3fn)
|
||||
attention, z_unfused, z_fused = _rope_quant_vs_unfused(
|
||||
attn, q, cache, metadata, torch.tensor(positions, device=device), num_tokens
|
||||
)
|
||||
|
||||
keys_all = cache.flatten(0, 1).float()
|
||||
for token, slots in enumerate(visible):
|
||||
keys = keys_all[slots.to(device)]
|
||||
weights = torch.softmax(q[token].float() @ keys.T * attn.scale, -1)
|
||||
torch.testing.assert_close(
|
||||
attention[token].float(), weights @ keys, atol=0.05, rtol=0.05
|
||||
)
|
||||
_assert_projects_alike(z_unfused, z_fused)
|
||||
@@ -1,310 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""RoPE equivalence tests for the fused Kimi-K3 MLA epilogues."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.models.kimi_k3.nvidia.ops.fused_mla_key_concat_kv_cache import (
|
||||
fused_mla_decode_q_concat_kv_cache_insert,
|
||||
fused_mla_key_concat_ds_mla_insert,
|
||||
fused_mla_key_concat_kv_cache_insert,
|
||||
fused_mla_kv_concat,
|
||||
fused_mla_kv_concat_quant_fp8,
|
||||
fused_mla_qkv_quant_kv_cache_fp8_insert,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="Kimi-K3 fused MLA requires CUDA"
|
||||
)
|
||||
|
||||
_DTYPE = torch.bfloat16
|
||||
_NUM_TOKENS = 3
|
||||
_NUM_HEADS = 4
|
||||
_BLOCK_SIZE = 8
|
||||
_POSITIONS = (1, 7, 13)
|
||||
_SLOTS = (0, 3, 9)
|
||||
|
||||
|
||||
def _randn(*shape: int, dtype: torch.dtype = _DTYPE) -> torch.Tensor:
|
||||
return torch.randn(*shape, device="cuda", dtype=dtype) * 0.2
|
||||
|
||||
|
||||
def _rope_cache(max_position: int = 32) -> torch.Tensor:
|
||||
inv_freq = 1.0 / (
|
||||
50000 ** (torch.arange(0, 64, 2, dtype=torch.float32, device="cuda") / 64)
|
||||
)
|
||||
positions = torch.arange(max_position, dtype=torch.float32, device="cuda")
|
||||
freqs = torch.outer(positions, inv_freq)
|
||||
# The fused epilogue reads the cos/sin table in fp32 (RoPE math runs in fp32).
|
||||
return torch.cat((freqs.cos(), freqs.sin()), dim=-1)
|
||||
|
||||
|
||||
def _apply_gptj_rope(
|
||||
x: torch.Tensor, positions: torch.Tensor, cos_sin_cache: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
cos, sin = cos_sin_cache.index_select(0, positions).chunk(2, dim=-1)
|
||||
for _ in range(x.ndim - 2):
|
||||
cos = cos.unsqueeze(1)
|
||||
sin = sin.unsqueeze(1)
|
||||
x1 = x[..., ::2].float()
|
||||
x2 = x[..., 1::2].float()
|
||||
out1 = x1 * cos.float() - x2 * sin.float()
|
||||
out2 = x2 * cos.float() + x1 * sin.float()
|
||||
return torch.stack((out1, out2), dim=-1).flatten(-2).to(x.dtype)
|
||||
|
||||
|
||||
def _cache_rows(cache: torch.Tensor, slots: torch.Tensor) -> torch.Tensor:
|
||||
return cache.reshape(-1, cache.shape[-1]).index_select(0, slots)
|
||||
|
||||
|
||||
def _assert_fp8_close(actual: torch.Tensor, expected: torch.Tensor) -> None:
|
||||
torch.testing.assert_close(
|
||||
actual.float(),
|
||||
expected.to(torch.float8_e4m3fn).float(),
|
||||
atol=0.03125,
|
||||
rtol=0.15,
|
||||
)
|
||||
|
||||
|
||||
def _strided_context_inputs(
|
||||
num_tokens: int, num_heads: int, dtype: torch.dtype, k_pe_fp8: bool
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""The real prefill-context layouts: ``k_nope``/``v`` as the two strided
|
||||
halves of one kv_b_proj output and ``k_pe`` as a column slice of the gather
|
||||
workspace (left in the fp8 cache layout for a plain fp8 cache)."""
|
||||
kv_nope = _randn(num_tokens, num_heads, 256, dtype=dtype)
|
||||
k_nope, v = kv_nope.split((128, 128), dim=-1)
|
||||
workspace = _randn(num_tokens, 576, dtype=dtype)
|
||||
if k_pe_fp8:
|
||||
workspace = workspace.to(torch.float8_e4m3fn)
|
||||
return k_nope, workspace[:, 512:].unsqueeze(1), v
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [0, _NUM_TOKENS])
|
||||
@pytest.mark.parametrize("num_heads", [3, _NUM_HEADS])
|
||||
@pytest.mark.parametrize("input_dtype", [torch.bfloat16, torch.float16])
|
||||
@torch.inference_mode()
|
||||
def test_context_kv_concat_accepts_strided_inputs(
|
||||
num_tokens: int, num_heads: int, input_dtype: torch.dtype
|
||||
) -> None:
|
||||
k_nope, k_pe, _ = _strided_context_inputs(
|
||||
num_tokens, num_heads, input_dtype, k_pe_fp8=False
|
||||
)
|
||||
|
||||
k = fused_mla_kv_concat(k_nope, k_pe)
|
||||
|
||||
assert k.is_contiguous()
|
||||
assert k.dtype == input_dtype
|
||||
torch.testing.assert_close(k[..., :128], k_nope, atol=0, rtol=0)
|
||||
torch.testing.assert_close(
|
||||
k[..., 128:], k_pe.expand(-1, num_heads, -1), atol=0, rtol=0
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("k_pe_dtype", ["input", "fp8"])
|
||||
@pytest.mark.parametrize("num_tokens", [0, _NUM_TOKENS])
|
||||
@pytest.mark.parametrize("num_heads", [3, _NUM_HEADS])
|
||||
@pytest.mark.parametrize("input_dtype", [torch.bfloat16, torch.float16])
|
||||
@torch.inference_mode()
|
||||
def test_context_kv_pack_quantizes_strided_inputs(
|
||||
k_pe_dtype: str,
|
||||
num_tokens: int,
|
||||
num_heads: int,
|
||||
input_dtype: torch.dtype,
|
||||
) -> None:
|
||||
k_nope, k_pe, v = _strided_context_inputs(
|
||||
num_tokens, num_heads, input_dtype, k_pe_fp8=k_pe_dtype == "fp8"
|
||||
)
|
||||
|
||||
k_actual, v_actual = fused_mla_kv_concat_quant_fp8(k_nope, k_pe, v)
|
||||
|
||||
fp8 = torch.float8_e4m3fn
|
||||
k_expected = torch.empty_like(k_actual)
|
||||
k_expected[..., :128] = k_nope.to(fp8)
|
||||
k_expected[..., 128:] = k_pe.to(fp8)
|
||||
assert k_actual.is_contiguous()
|
||||
assert v_actual.is_contiguous()
|
||||
# The pack casts with the native pairwise converters, so it must be
|
||||
# bit-identical to torch's `.to(fp8)` rather than merely close.
|
||||
torch.testing.assert_close(k_actual.float(), k_expected.float(), atol=0, rtol=0)
|
||||
torch.testing.assert_close(v_actual.float(), v.to(fp8).float(), atol=0, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cache_kind", ["bf16", "fp8", "fp8_ds_mla"])
|
||||
@torch.inference_mode()
|
||||
def test_prefill_epilogue_fuses_gptj_rope(cache_kind: str) -> None:
|
||||
torch.manual_seed(0)
|
||||
positions = torch.tensor(_POSITIONS, device="cuda", dtype=torch.int64)
|
||||
slots = torch.tensor(_SLOTS, device="cuda", dtype=torch.int64)
|
||||
cos_sin_cache = _rope_cache()
|
||||
q = _randn(_NUM_TOKENS, _NUM_HEADS, 192)
|
||||
k_nope = _randn(_NUM_TOKENS, _NUM_HEADS, 128)
|
||||
k_pe = _randn(_NUM_TOKENS, 64)
|
||||
kv_c = _randn(_NUM_TOKENS, 512)
|
||||
v = _randn(_NUM_TOKENS, _NUM_HEADS, 128)
|
||||
|
||||
q_expected = q.clone()
|
||||
q_expected[..., 128:] = _apply_gptj_rope(
|
||||
q_expected[..., 128:], positions, cos_sin_cache
|
||||
)
|
||||
k_pe_expected = _apply_gptj_rope(k_pe, positions, cos_sin_cache)
|
||||
k_expected = torch.cat(
|
||||
(k_nope, k_pe_expected[:, None, :].expand(-1, _NUM_HEADS, -1)), dim=-1
|
||||
)
|
||||
cache_expected = torch.cat((kv_c, k_pe_expected), dim=-1)
|
||||
|
||||
if cache_kind == "bf16":
|
||||
cache = torch.zeros(2, _BLOCK_SIZE, 576, device="cuda", dtype=_DTYPE)
|
||||
q_actual = q.clone()
|
||||
k_actual = fused_mla_key_concat_kv_cache_insert(
|
||||
q_actual,
|
||||
k_nope,
|
||||
k_pe,
|
||||
kv_c,
|
||||
cache,
|
||||
slots,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
)
|
||||
torch.testing.assert_close(q_actual, q_expected)
|
||||
torch.testing.assert_close(k_actual, k_expected)
|
||||
torch.testing.assert_close(_cache_rows(cache, slots), cache_expected)
|
||||
elif cache_kind == "fp8":
|
||||
cache = torch.zeros(
|
||||
2, _BLOCK_SIZE, 576, device="cuda", dtype=torch.float8_e4m3fn
|
||||
)
|
||||
one = torch.ones(1, device="cuda", dtype=torch.float32)
|
||||
q_actual, k_actual, v_actual = fused_mla_qkv_quant_kv_cache_fp8_insert(
|
||||
q,
|
||||
k_nope,
|
||||
k_pe,
|
||||
kv_c,
|
||||
v,
|
||||
cache,
|
||||
slots,
|
||||
one,
|
||||
one,
|
||||
one,
|
||||
one,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
)
|
||||
_assert_fp8_close(q_actual, q_expected)
|
||||
_assert_fp8_close(k_actual, k_expected)
|
||||
_assert_fp8_close(v_actual, v)
|
||||
_assert_fp8_close(_cache_rows(cache, slots), cache_expected)
|
||||
else:
|
||||
cache = torch.zeros(2, _BLOCK_SIZE, 656, device="cuda", dtype=torch.uint8)
|
||||
q_actual = q.clone()
|
||||
k_actual = fused_mla_key_concat_ds_mla_insert(
|
||||
q_actual,
|
||||
k_nope,
|
||||
k_pe,
|
||||
kv_c,
|
||||
cache,
|
||||
slots,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
)
|
||||
rope_cache = _cache_rows(cache, slots)[:, 528:656].view(_DTYPE)
|
||||
torch.testing.assert_close(q_actual, q_expected)
|
||||
torch.testing.assert_close(k_actual, k_expected)
|
||||
torch.testing.assert_close(rope_cache, k_pe_expected)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cache_kind", ["bf16", "fp8", "fp8_ds_mla"])
|
||||
@torch.inference_mode()
|
||||
def test_decode_epilogue_fuses_gptj_rope(cache_kind: str) -> None:
|
||||
torch.manual_seed(1)
|
||||
positions = torch.tensor(_POSITIONS, device="cuda", dtype=torch.int64)
|
||||
slots = torch.tensor(_SLOTS, device="cuda", dtype=torch.int64)
|
||||
cos_sin_cache = _rope_cache()
|
||||
ql_nope = _randn(_NUM_TOKENS, _NUM_HEADS, 512)
|
||||
q_pe = _randn(_NUM_TOKENS, _NUM_HEADS, 64)
|
||||
kv_c = _randn(_NUM_TOKENS, 512)
|
||||
k_pe = _randn(_NUM_TOKENS, 64)
|
||||
|
||||
q_pe_expected = _apply_gptj_rope(q_pe, positions, cos_sin_cache)
|
||||
k_pe_expected = _apply_gptj_rope(k_pe, positions, cos_sin_cache)
|
||||
q_expected = torch.cat((ql_nope, q_pe_expected), dim=-1)
|
||||
cache_expected = torch.cat((kv_c, k_pe_expected), dim=-1)
|
||||
|
||||
kwargs = {"positions": positions, "cos_sin_cache": cos_sin_cache}
|
||||
if cache_kind == "bf16":
|
||||
cache = torch.zeros(2, _BLOCK_SIZE, 576, device="cuda", dtype=_DTYPE)
|
||||
q_actual = fused_mla_decode_q_concat_kv_cache_insert(
|
||||
ql_nope, q_pe, kv_c, k_pe, cache, slots, **kwargs
|
||||
)
|
||||
torch.testing.assert_close(q_actual, q_expected)
|
||||
torch.testing.assert_close(_cache_rows(cache, slots), cache_expected)
|
||||
elif cache_kind == "fp8":
|
||||
cache = torch.zeros(
|
||||
2, _BLOCK_SIZE, 576, device="cuda", dtype=torch.float8_e4m3fn
|
||||
)
|
||||
one = torch.ones(1, device="cuda", dtype=torch.float32)
|
||||
q_actual = fused_mla_decode_q_concat_kv_cache_insert(
|
||||
ql_nope,
|
||||
q_pe,
|
||||
kv_c,
|
||||
k_pe,
|
||||
cache,
|
||||
slots,
|
||||
q_scale_inv=one,
|
||||
cache_scale_inv=one,
|
||||
**kwargs,
|
||||
)
|
||||
_assert_fp8_close(q_actual, q_expected)
|
||||
_assert_fp8_close(_cache_rows(cache, slots), cache_expected)
|
||||
else:
|
||||
cache = torch.zeros(2, _BLOCK_SIZE, 656, device="cuda", dtype=torch.uint8)
|
||||
q_actual = fused_mla_decode_q_concat_kv_cache_insert(
|
||||
ql_nope, q_pe, kv_c, k_pe, cache, slots, ds_mla=True, **kwargs
|
||||
)
|
||||
rope_cache = _cache_rows(cache, slots)[:, 528:656].view(_DTYPE)
|
||||
torch.testing.assert_close(q_actual, q_expected)
|
||||
torch.testing.assert_close(rope_cache, k_pe_expected)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_decode_epilogue_preserves_nope_path() -> None:
|
||||
torch.manual_seed(2)
|
||||
slots = torch.tensor(_SLOTS, device="cuda", dtype=torch.int64)
|
||||
ql_nope = _randn(_NUM_TOKENS, _NUM_HEADS, 512)
|
||||
q_pe = _randn(_NUM_TOKENS, _NUM_HEADS, 64)
|
||||
kv_c = _randn(_NUM_TOKENS, 512)
|
||||
k_pe = _randn(_NUM_TOKENS, 64)
|
||||
cache = torch.zeros(2, _BLOCK_SIZE, 576, device="cuda", dtype=_DTYPE)
|
||||
|
||||
q_actual = fused_mla_decode_q_concat_kv_cache_insert(
|
||||
ql_nope, q_pe, kv_c, k_pe, cache, slots
|
||||
)
|
||||
|
||||
torch.testing.assert_close(q_actual, torch.cat((ql_nope, q_pe), dim=-1))
|
||||
torch.testing.assert_close(
|
||||
_cache_rows(cache, slots), torch.cat((kv_c, k_pe), dim=-1)
|
||||
)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
@pytest.mark.parametrize("num_tokens", [64, 65])
|
||||
def test_decode_epilogue_row_split_boundary(num_tokens: int) -> None:
|
||||
"""Both sides of the M-based warps-per-row dispatch produce the same rows."""
|
||||
torch.manual_seed(3)
|
||||
num_blocks = (num_tokens + _BLOCK_SIZE - 1) // _BLOCK_SIZE
|
||||
slots = torch.randperm(num_blocks * _BLOCK_SIZE, device="cuda")[:num_tokens].long()
|
||||
ql_nope = _randn(num_tokens, _NUM_HEADS, 512)
|
||||
q_pe = _randn(num_tokens, _NUM_HEADS, 64)
|
||||
kv_c = _randn(num_tokens, 512)
|
||||
k_pe = _randn(num_tokens, 64)
|
||||
cache = torch.zeros(num_blocks, _BLOCK_SIZE, 576, device="cuda", dtype=_DTYPE)
|
||||
|
||||
q_actual = fused_mla_decode_q_concat_kv_cache_insert(
|
||||
ql_nope, q_pe, kv_c, k_pe, cache, slots
|
||||
)
|
||||
|
||||
torch.testing.assert_close(q_actual, torch.cat((ql_nope, q_pe), dim=-1))
|
||||
torch.testing.assert_close(
|
||||
_cache_rows(cache, slots), torch.cat((kv_c, k_pe), dim=-1)
|
||||
)
|
||||
@@ -1,257 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm._custom_ops as ops
|
||||
from vllm.models.kimi_k3.nvidia.ops.fused_mla_key_concat_kv_cache import (
|
||||
fused_mla_decode_q_concat_kv_cache_insert,
|
||||
fused_mla_key_concat_ds_mla_insert,
|
||||
fused_mla_key_concat_kv_cache_insert,
|
||||
fused_mla_qkv_quant_kv_cache_fp8_insert,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
NUM_TOKENS = 4
|
||||
NUM_HEADS = 2
|
||||
KV_LORA_RANK = 512
|
||||
NOPE_HEAD_DIM = 128
|
||||
ROPE_HEAD_DIM = 64
|
||||
QK_HEAD_DIM = NOPE_HEAD_DIM + ROPE_HEAD_DIM
|
||||
V_HEAD_DIM = 128
|
||||
CACHE_ENTRY = KV_LORA_RANK + ROPE_HEAD_DIM
|
||||
DS_MLA_CACHE_ENTRY = 656
|
||||
BLOCK_SIZE = 8
|
||||
OWNED_TOKEN_INDICES = [1, 3]
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="Kimi-K3 fused MLA ops require CUDA"
|
||||
)
|
||||
|
||||
|
||||
def _inputs() -> dict[str, torch.Tensor]:
|
||||
torch.manual_seed(0)
|
||||
|
||||
def make(*shape: int) -> torch.Tensor:
|
||||
return torch.randn(*shape, device="cuda", dtype=torch.bfloat16)
|
||||
|
||||
return {
|
||||
"q": make(NUM_TOKENS, NUM_HEADS, QK_HEAD_DIM),
|
||||
"k_nope": make(NUM_TOKENS, NUM_HEADS, NOPE_HEAD_DIM),
|
||||
"k_pe": make(NUM_TOKENS, ROPE_HEAD_DIM),
|
||||
"kv_c": make(NUM_TOKENS, KV_LORA_RANK),
|
||||
"v": make(NUM_TOKENS, NUM_HEADS, V_HEAD_DIM),
|
||||
"ql_nope": make(NUM_TOKENS, NUM_HEADS, KV_LORA_RANK),
|
||||
"q_pe": make(NUM_TOKENS, NUM_HEADS, ROPE_HEAD_DIM),
|
||||
}
|
||||
|
||||
|
||||
def _slot_mapping() -> torch.Tensor:
|
||||
return torch.tensor([-1, 0, -1, 5], device="cuda", dtype=torch.int64)
|
||||
|
||||
|
||||
def _owned(tensor: torch.Tensor) -> torch.Tensor:
|
||||
return tensor[OWNED_TOKEN_INDICES]
|
||||
|
||||
|
||||
def _full_key(k_nope: torch.Tensor, k_pe: torch.Tensor) -> torch.Tensor:
|
||||
shared_pe = k_pe[:, None, :].expand(-1, k_nope.shape[1], -1)
|
||||
return torch.cat((k_nope, shared_pe), dim=-1)
|
||||
|
||||
|
||||
def _latent_query(ql_nope: torch.Tensor, q_pe: torch.Tensor) -> torch.Tensor:
|
||||
return torch.cat((ql_nope, q_pe), dim=-1)
|
||||
|
||||
|
||||
def _make_cache(cache_format: str) -> torch.Tensor:
|
||||
if cache_format == "bf16":
|
||||
return torch.full(
|
||||
(1, BLOCK_SIZE, CACHE_ENTRY),
|
||||
-7,
|
||||
device="cuda",
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
if cache_format == "fp8":
|
||||
return torch.full(
|
||||
(1, BLOCK_SIZE, CACHE_ENTRY),
|
||||
1.0,
|
||||
device="cuda",
|
||||
dtype=torch.float32,
|
||||
).to(torch.float8_e4m3fn)
|
||||
assert cache_format == "fp8_ds_mla"
|
||||
return torch.full(
|
||||
(1, BLOCK_SIZE, DS_MLA_CACHE_ENTRY),
|
||||
165,
|
||||
device="cuda",
|
||||
dtype=torch.uint8,
|
||||
)
|
||||
|
||||
|
||||
def _reference_cache(
|
||||
cache_format: str,
|
||||
inputs: dict[str, torch.Tensor],
|
||||
initial_cache: torch.Tensor,
|
||||
slots: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
reference_cache = initial_cache.clone()
|
||||
valid_slots = _owned(slots)
|
||||
if cache_format == "fp8_ds_mla":
|
||||
ops.concat_and_cache_mla(
|
||||
_owned(inputs["kv_c"]),
|
||||
_owned(inputs["k_pe"]),
|
||||
reference_cache,
|
||||
valid_slots,
|
||||
cache_format,
|
||||
torch.ones(1, device="cuda", dtype=torch.float32),
|
||||
)
|
||||
return reference_cache
|
||||
|
||||
latent = torch.cat(
|
||||
(_owned(inputs["kv_c"]), _owned(inputs["k_pe"])),
|
||||
dim=-1,
|
||||
)
|
||||
flat_reference_cache = reference_cache.view(-1, CACHE_ENTRY)
|
||||
for row, slot in zip(latent, valid_slots.tolist()):
|
||||
flat_reference_cache[slot].copy_(row.to(reference_cache.dtype))
|
||||
return reference_cache
|
||||
|
||||
|
||||
def _assert_cache_matches_reference(
|
||||
cache: torch.Tensor,
|
||||
reference_cache: torch.Tensor,
|
||||
initial_cache: torch.Tensor,
|
||||
) -> None:
|
||||
assert torch.equal(cache, reference_cache)
|
||||
for slot in (0, 5):
|
||||
assert not torch.equal(cache[0, slot], initial_cache[0, slot])
|
||||
for slot in (1, 2, 3, 4, 6, 7):
|
||||
assert torch.equal(cache[0, slot], initial_cache[0, slot])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cache_format", ["bf16", "fp8_ds_mla"])
|
||||
def test_prefill_concat_ignores_negative_slots_for_cache(cache_format: str) -> None:
|
||||
inputs = _inputs()
|
||||
slots = _slot_mapping()
|
||||
initial_cache = _make_cache(cache_format)
|
||||
mixed_cache = initial_cache.clone()
|
||||
reference_cache = _reference_cache(cache_format, inputs, initial_cache, slots)
|
||||
|
||||
op = (
|
||||
fused_mla_key_concat_kv_cache_insert
|
||||
if cache_format == "bf16"
|
||||
else fused_mla_key_concat_ds_mla_insert
|
||||
)
|
||||
output = op(
|
||||
inputs["q"],
|
||||
inputs["k_nope"],
|
||||
inputs["k_pe"],
|
||||
inputs["kv_c"],
|
||||
mixed_cache,
|
||||
slots,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(
|
||||
output, _full_key(inputs["k_nope"], inputs["k_pe"]), rtol=0, atol=0
|
||||
)
|
||||
_assert_cache_matches_reference(mixed_cache, reference_cache, initial_cache)
|
||||
|
||||
|
||||
def test_fp8_prefill_outputs_ignore_negative_slots_for_cache() -> None:
|
||||
inputs = _inputs()
|
||||
slots = _slot_mapping()
|
||||
initial_cache = _make_cache("fp8")
|
||||
mixed_cache = initial_cache.clone()
|
||||
reference_cache = _reference_cache("fp8", inputs, initial_cache, slots)
|
||||
scale_inv = torch.ones(1, device="cuda", dtype=torch.float32)
|
||||
|
||||
q, k, v = fused_mla_qkv_quant_kv_cache_fp8_insert(
|
||||
inputs["q"],
|
||||
inputs["k_nope"],
|
||||
inputs["k_pe"],
|
||||
inputs["kv_c"],
|
||||
inputs["v"],
|
||||
mixed_cache,
|
||||
slots,
|
||||
scale_inv,
|
||||
scale_inv,
|
||||
scale_inv,
|
||||
scale_inv,
|
||||
)
|
||||
|
||||
expected = (
|
||||
inputs["q"].to(torch.float8_e4m3fn),
|
||||
_full_key(inputs["k_nope"], inputs["k_pe"]).to(torch.float8_e4m3fn),
|
||||
inputs["v"].to(torch.float8_e4m3fn),
|
||||
)
|
||||
for actual, reference in zip((q, k, v), expected):
|
||||
torch.testing.assert_close(actual.float(), reference.float(), rtol=0, atol=0)
|
||||
_assert_cache_matches_reference(mixed_cache, reference_cache, initial_cache)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cache_format", ["bf16", "fp8", "fp8_ds_mla"])
|
||||
def test_decode_concat_ignores_negative_slots_for_cache(cache_format: str) -> None:
|
||||
inputs = _inputs()
|
||||
slots = _slot_mapping()
|
||||
initial_cache = _make_cache(cache_format)
|
||||
mixed_cache = initial_cache.clone()
|
||||
reference_cache = _reference_cache(cache_format, inputs, initial_cache, slots)
|
||||
scale_inv = torch.ones(1, device="cuda", dtype=torch.float32)
|
||||
|
||||
output = fused_mla_decode_q_concat_kv_cache_insert(
|
||||
inputs["ql_nope"],
|
||||
inputs["q_pe"],
|
||||
inputs["kv_c"],
|
||||
inputs["k_pe"],
|
||||
mixed_cache,
|
||||
slots,
|
||||
ds_mla=cache_format == "fp8_ds_mla",
|
||||
q_scale_inv=scale_inv if cache_format == "fp8" else None,
|
||||
cache_scale_inv=scale_inv if cache_format == "fp8" else None,
|
||||
)
|
||||
|
||||
expected = _latent_query(inputs["ql_nope"], inputs["q_pe"])
|
||||
if cache_format == "fp8":
|
||||
expected = expected.to(torch.float8_e4m3fn)
|
||||
torch.testing.assert_close(output.float(), expected.float(), rtol=0, atol=0)
|
||||
else:
|
||||
torch.testing.assert_close(output, expected, rtol=0, atol=0)
|
||||
_assert_cache_matches_reference(mixed_cache, reference_cache, initial_cache)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("decode", [False, True])
|
||||
def test_ds_mla_cache_insert_bit_compatible_with_reference(decode: bool) -> None:
|
||||
"""Fused ds_mla insertion must match the reference bit-for-bit."""
|
||||
torch.manual_seed(0)
|
||||
num_tokens, num_blocks = 33, 16
|
||||
dt = torch.bfloat16
|
||||
kv_c = torch.randn(num_tokens, KV_LORA_RANK, device="cuda", dtype=dt)
|
||||
k_pe = torch.randn(num_tokens, 1, ROPE_HEAD_DIM, device="cuda", dtype=dt)
|
||||
k_nope = torch.randn(num_tokens, NUM_HEADS, NOPE_HEAD_DIM, device="cuda", dtype=dt)
|
||||
q = torch.randn(num_tokens, NUM_HEADS, QK_HEAD_DIM, device="cuda", dtype=dt)
|
||||
slots = torch.randperm(num_blocks * BLOCK_SIZE, device="cuda")[:num_tokens].long()
|
||||
slots[torch.rand(num_tokens, device="cuda") < 0.3] = -1
|
||||
ref = torch.full(
|
||||
(num_blocks, BLOCK_SIZE, DS_MLA_CACHE_ENTRY),
|
||||
165,
|
||||
device="cuda",
|
||||
dtype=torch.uint8,
|
||||
)
|
||||
got = ref.clone()
|
||||
scale = torch.ones(1, device="cuda", dtype=torch.float32)
|
||||
ops.concat_and_cache_mla(kv_c, k_pe.squeeze(1), ref, slots, "fp8_ds_mla", scale)
|
||||
if decode:
|
||||
ql_nope = torch.randn(
|
||||
num_tokens, NUM_HEADS, KV_LORA_RANK, device="cuda", dtype=dt
|
||||
)
|
||||
fused_mla_decode_q_concat_kv_cache_insert(
|
||||
ql_nope, q[..., -ROPE_HEAD_DIM:], kv_c, k_pe, got, slots, ds_mla=True
|
||||
)
|
||||
else:
|
||||
fused_mla_key_concat_ds_mla_insert(q, k_nope, k_pe, kv_c, got, slots)
|
||||
assert torch.equal(ref, got)
|
||||
rows = got.view(-1, DS_MLA_CACHE_ENTRY)[slots[slots >= 0]]
|
||||
scales = rows[:, KV_LORA_RANK : KV_LORA_RANK + 16].view(torch.float32)
|
||||
torch.testing.assert_close(
|
||||
torch.log2(scales), torch.log2(scales).round(), rtol=0, atol=0
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,143 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""fp8 query and index-K cache on the Triton indexer must agree with bf16.
|
||||
|
||||
The MiniMax M3 lightning-indexer scores only feed a top-k block ranking (the
|
||||
attention itself reads the main KV cache), so an fp8 (e4m3) side cache trades
|
||||
~2% relative score error for half the per-step index-K read bandwidth — the
|
||||
dominant linear-in-context decode cost at long context. #45892 enabled fp8
|
||||
index caches on the SM100 MSA path; this covers the Triton impl used
|
||||
everywhere else (e.g. SM120), for both the decode and prefill scorers.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.models.minimax_m3.common.ops.index_topk import (
|
||||
SPARSE_BLOCK_SIZE,
|
||||
minimax_m3_index_decode,
|
||||
minimax_m3_index_score,
|
||||
minimax_m3_index_topk,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
FP8_CUDA_SUPPORTED = current_platform.is_cuda() and current_platform.supports_fp8()
|
||||
PAGE = SPARSE_BLOCK_SIZE
|
||||
HEAD_DIM = 128
|
||||
HEADS = 1
|
||||
TOPK = 16
|
||||
NUM_PAGES = 100
|
||||
SEQ_LEN = 96 * PAGE + 37
|
||||
PLANTED = (7, 40, 88)
|
||||
|
||||
|
||||
def _make_inputs(device: str) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
torch.manual_seed(3)
|
||||
keys = torch.randn(NUM_PAGES * PAGE, HEAD_DIM, dtype=torch.bfloat16, device=device)
|
||||
# RMS-normalize like the real (post index_k_norm) keys.
|
||||
keys = keys * keys.float().pow(2).mean(-1, keepdim=True).add(1e-6).rsqrt().to(
|
||||
torch.bfloat16
|
||||
)
|
||||
q = torch.randn(1, HEADS, HEAD_DIM, dtype=torch.bfloat16, device=device)
|
||||
for b in PLANTED:
|
||||
keys[b * PAGE + 5] = (q[0, 0] * 3).to(torch.bfloat16)
|
||||
return q, keys
|
||||
|
||||
|
||||
def _check_selection(top_bf16: set[int], top_fp8: set[int]) -> None:
|
||||
# Planted (clearly relevant) blocks must never be lost.
|
||||
assert all(b in top_fp8 for b in PLANTED)
|
||||
# Selection may differ only at the noise floor (near-tied filler blocks).
|
||||
assert len(top_bf16 & top_fp8) >= TOPK - 2
|
||||
|
||||
|
||||
@pytest.mark.skipif(not FP8_CUDA_SUPPORTED, reason="CUDA FP8 support required")
|
||||
@torch.inference_mode()
|
||||
def test_fp8_index_query_and_cache_decode_topk_matches_bf16() -> None:
|
||||
dev = "cuda"
|
||||
q, keys = _make_inputs(dev)
|
||||
block_table = torch.arange(NUM_PAGES, dtype=torch.int32, device=dev).unsqueeze(0)
|
||||
seq_lens = torch.tensor([SEQ_LEN], dtype=torch.int32, device=dev)
|
||||
|
||||
def select(cache: torch.Tensor) -> set[int]:
|
||||
query = q.to(cache.dtype)
|
||||
idx = minimax_m3_index_decode(
|
||||
query,
|
||||
cache.view(NUM_PAGES, PAGE, HEAD_DIM).contiguous(),
|
||||
block_table,
|
||||
seq_lens,
|
||||
SEQ_LEN,
|
||||
TOPK,
|
||||
0,
|
||||
0,
|
||||
HEADS,
|
||||
1,
|
||||
1,
|
||||
)
|
||||
return {int(b) for b in idx[0, 0].tolist() if b >= 0}
|
||||
|
||||
_check_selection(select(keys), select(keys.to(torch.float8_e4m3fn)))
|
||||
|
||||
|
||||
@pytest.mark.skipif(not FP8_CUDA_SUPPORTED, reason="CUDA FP8 support required")
|
||||
@torch.inference_mode()
|
||||
def test_fp8_index_query_and_cache_prefill_topk_matches_bf16() -> None:
|
||||
dev = "cuda"
|
||||
q, keys = _make_inputs(dev)
|
||||
block_table = torch.arange(NUM_PAGES, dtype=torch.int32, device=dev).unsqueeze(0)
|
||||
seq_lens = torch.tensor([SEQ_LEN], dtype=torch.int32, device=dev)
|
||||
# single decode-like query token at the end of the sequence
|
||||
cu_seqlens_q = torch.tensor([0, 1], dtype=torch.int32, device=dev)
|
||||
prefix_lens = torch.tensor([SEQ_LEN - 1], dtype=torch.int32, device=dev)
|
||||
|
||||
def select(cache: torch.Tensor) -> set[int]:
|
||||
query = q.to(cache.dtype)
|
||||
score = minimax_m3_index_score(
|
||||
query,
|
||||
cache.view(NUM_PAGES, PAGE, HEAD_DIM).contiguous(),
|
||||
block_table,
|
||||
cu_seqlens_q,
|
||||
seq_lens,
|
||||
prefix_lens,
|
||||
1,
|
||||
SEQ_LEN,
|
||||
HEADS,
|
||||
)
|
||||
idx = minimax_m3_index_topk(score, cu_seqlens_q, prefix_lens, 1, TOPK, 0, 0)
|
||||
return {int(b) for b in idx[0, 0].tolist() if b >= 0}
|
||||
|
||||
_check_selection(select(keys), select(keys.to(torch.float8_e4m3fn)))
|
||||
|
||||
|
||||
@pytest.mark.skipif(not FP8_CUDA_SUPPORTED, reason="CUDA FP8 support required")
|
||||
@torch.inference_mode()
|
||||
def test_bf16_query_against_fp8_cache_decode_topk_matches_bf16() -> None:
|
||||
"""A mixed pair must upcast rather than emit a mixed-dtype fp8 dot.
|
||||
|
||||
Production allocates index_q with the index-K cache dtype, so the scorers
|
||||
normally see a matched pair and keep the native FP8 MMA. This pins the
|
||||
other branch: a bf16 query against an e4m3 cache still compiles and still
|
||||
ranks the planted blocks.
|
||||
"""
|
||||
dev = "cuda"
|
||||
q, keys = _make_inputs(dev)
|
||||
block_table = torch.arange(NUM_PAGES, dtype=torch.int32, device=dev).unsqueeze(0)
|
||||
seq_lens = torch.tensor([SEQ_LEN], dtype=torch.int32, device=dev)
|
||||
|
||||
def select(cache: torch.Tensor) -> set[int]:
|
||||
idx = minimax_m3_index_decode(
|
||||
q, # bf16 query, deliberately not cast to the cache dtype
|
||||
cache.view(NUM_PAGES, PAGE, HEAD_DIM).contiguous(),
|
||||
block_table,
|
||||
seq_lens,
|
||||
SEQ_LEN,
|
||||
TOPK,
|
||||
0,
|
||||
0,
|
||||
HEADS,
|
||||
1,
|
||||
1,
|
||||
)
|
||||
return {int(b) for b in idx[0, 0].tolist() if b >= 0}
|
||||
|
||||
_check_selection(select(keys), select(keys.to(torch.float8_e4m3fn)))
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,226 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""ROCm correctness tests for the strided DSA candidate mask.
|
||||
|
||||
``_apply_candidate_mask_strided`` is a decode-path variant of the shared
|
||||
``apply_candidate_mask``. It differs in two ways that need pinning down:
|
||||
|
||||
* it launches a fixed number of programs that stride the column axis, rather
|
||||
than one program per 1024-column tile, so at ``max_model_len`` widths the
|
||||
grid no longer scales with a workspace that is mostly padding;
|
||||
* it stops at each row's ``end`` instead of sanitizing the whole width.
|
||||
|
||||
The second is only sound because every consumer bounds its scan by the same
|
||||
ends -- ``top_k_per_row_decode`` takes ``rowEnd`` from ``seq_lens``. So the
|
||||
contract is agreement with the shared kernel over ``[0, end)`` and nothing
|
||||
beyond it, which is what these tests assert rather than comparing whole rows.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_rocm(), reason="ROCm-specific tests"
|
||||
)
|
||||
|
||||
BLOCK_SIZE = 8
|
||||
TOPK_BLOCKS = 64
|
||||
# Above 128 tiles the grid saturates and programs start striding; below it the
|
||||
# min() clamp applies instead. Both regimes need covering, so the widths here
|
||||
# deliberately straddle 128 * 1024.
|
||||
NARROW = 8192
|
||||
WIDE = 262144
|
||||
|
||||
|
||||
def _inputs(rows, width, ends, starts, *, block_size, col_stride, dtype):
|
||||
"""Logits, bounds and candidate blocks shared by both kernels."""
|
||||
torch.manual_seed(0)
|
||||
# A column stride > 1 keeps the kernels honest about stride_col; the decode
|
||||
# caller slices a workspace, so contiguous rows are not guaranteed.
|
||||
logits = torch.randn(rows, width * col_stride, device="cuda")[:, ::col_stride]
|
||||
logits[0, : min(16, width)] = 0.0
|
||||
if rows > 2:
|
||||
logits[2, 5] = float("nan")
|
||||
row_ke = torch.tensor(ends, device="cuda", dtype=dtype)
|
||||
row_ks = torch.tensor(starts, device="cuda", dtype=dtype) if starts else None
|
||||
nblocks = (width + block_size - 1) // block_size
|
||||
candidates = torch.randint(
|
||||
0, nblocks, (rows, TOPK_BLOCKS), device="cuda", dtype=torch.int32
|
||||
)
|
||||
# -1 is the "no block" sentinel the selector emits for short rows.
|
||||
candidates[:, ::7] = -1
|
||||
return logits, row_ks, row_ke, candidates
|
||||
|
||||
|
||||
def _run_both(logits, row_ks, row_ke, candidates, block_size, row_repeat):
|
||||
from vllm.model_executor.kernels.attention.dsa.candidate_blocks import (
|
||||
apply_candidate_mask,
|
||||
)
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
_apply_candidate_mask_strided,
|
||||
)
|
||||
|
||||
ref = logits.clone()
|
||||
got = logits.clone()
|
||||
apply_candidate_mask(ref, row_ks, row_ke, candidates, block_size, row_repeat)
|
||||
_apply_candidate_mask_strided(
|
||||
got, row_ks, row_ke, candidates, block_size, row_repeat
|
||||
)
|
||||
return ref, got
|
||||
|
||||
|
||||
def _assert_agrees_within_ends(ref, got, row_ke, row_repeat):
|
||||
"""Compare only the region a consumer can read."""
|
||||
for row in range(ref.shape[0]):
|
||||
end = int(row_ke[row // row_repeat])
|
||||
if end <= 0:
|
||||
continue
|
||||
torch.testing.assert_close(
|
||||
got[row, :end],
|
||||
ref[row, :end],
|
||||
rtol=0,
|
||||
atol=0,
|
||||
equal_nan=True,
|
||||
msg=lambda m, r=row, e=end: f"row {r} differs within [0, {e}): {m}",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("width", [NARROW, WIDE])
|
||||
@pytest.mark.parametrize("col_stride", [1, 2])
|
||||
def test_matches_shared_kernel_ragged_ends(width, col_stride):
|
||||
"""Ends on and around tile boundaries, plus the empty and full rows."""
|
||||
ends = [0, 1, 1023, 1024, 1025, width - 1, width, width // 2]
|
||||
logits, row_ks, row_ke, candidates = _inputs(
|
||||
len(ends),
|
||||
width,
|
||||
ends,
|
||||
None,
|
||||
block_size=BLOCK_SIZE,
|
||||
col_stride=col_stride,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
ref, got = _run_both(logits, row_ks, row_ke, candidates, BLOCK_SIZE, 1)
|
||||
_assert_agrees_within_ends(ref, got, row_ke, 1)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("block_size", [8, 16])
|
||||
def test_matches_shared_kernel_with_starts(block_size):
|
||||
"""Non-zero starts: the decode caller passes zeros, prefill does not."""
|
||||
ends = [0, 64, 4096, 8192]
|
||||
starts = [0, 7, 1024, 3]
|
||||
logits, row_ks, row_ke, candidates = _inputs(
|
||||
len(ends),
|
||||
NARROW,
|
||||
ends,
|
||||
starts,
|
||||
block_size=block_size,
|
||||
col_stride=1,
|
||||
dtype=torch.int64,
|
||||
)
|
||||
ref, got = _run_both(logits, row_ks, row_ke, candidates, block_size, 1)
|
||||
_assert_agrees_within_ends(ref, got, row_ke, 1)
|
||||
|
||||
|
||||
def test_matches_shared_kernel_row_repeat():
|
||||
"""Speculative decode: next_n logit rows share one bound."""
|
||||
row_repeat = 3
|
||||
ends = [0, 1025, WIDE]
|
||||
logits, row_ks, row_ke, candidates = _inputs(
|
||||
len(ends) * row_repeat,
|
||||
WIDE,
|
||||
ends,
|
||||
None,
|
||||
block_size=BLOCK_SIZE,
|
||||
col_stride=1,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
ref, got = _run_both(logits, row_ks, row_ke, candidates, BLOCK_SIZE, row_repeat)
|
||||
_assert_agrees_within_ends(ref, got, row_ke, row_repeat)
|
||||
|
||||
|
||||
def test_leaves_columns_past_end_untouched():
|
||||
"""The deliberate difference from the shared kernel, pinned explicitly.
|
||||
|
||||
Skipping the tail is where the speedup comes from, so a change that
|
||||
quietly restores full-width sanitizing should fail here rather than just
|
||||
get slower. Work stops at the tile boundary containing ``end``, not at
|
||||
``end`` itself, so that is the bound asserted.
|
||||
"""
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
_MASK_TILE,
|
||||
_apply_candidate_mask_strided,
|
||||
)
|
||||
|
||||
ends = [0, 1, 1025, WIDE // 2]
|
||||
logits, row_ks, row_ke, candidates = _inputs(
|
||||
len(ends),
|
||||
WIDE,
|
||||
ends,
|
||||
None,
|
||||
block_size=BLOCK_SIZE,
|
||||
col_stride=1,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
sentinel = 1.5
|
||||
logits.fill_(sentinel)
|
||||
_apply_candidate_mask_strided(logits, row_ks, row_ke, candidates, BLOCK_SIZE, 1)
|
||||
for row, end in enumerate(ends):
|
||||
touched = min(-(-end // _MASK_TILE) * _MASK_TILE, WIDE)
|
||||
tail = logits[row, touched:]
|
||||
assert torch.equal(tail, torch.full_like(tail, sentinel)), (
|
||||
f"row {row} (end {end}) was written past column {touched}"
|
||||
)
|
||||
# And the bound is tight: a row that ends early must not have cost a
|
||||
# full-width pass, which is the whole point of the kernel.
|
||||
assert touched <= end + _MASK_TILE
|
||||
|
||||
|
||||
def test_cudagraph_replay_tracks_changing_ends():
|
||||
"""The grid is static so a FULL capture stays valid when ends change.
|
||||
|
||||
Capturing bakes the grid, so a data-dependent program count would freeze
|
||||
whatever lengths were resident at capture. Replaying against longer and
|
||||
shorter contexts than the captured ones is the property that matters.
|
||||
"""
|
||||
from vllm.model_executor.kernels.attention.dsa.candidate_blocks import (
|
||||
apply_candidate_mask,
|
||||
)
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
_apply_candidate_mask_strided,
|
||||
)
|
||||
|
||||
rows = 4
|
||||
captured_ends = [1024, 4096, 8192, 2048]
|
||||
logits, row_ks, row_ke, candidates = _inputs(
|
||||
rows,
|
||||
WIDE,
|
||||
captured_ends,
|
||||
None,
|
||||
block_size=BLOCK_SIZE,
|
||||
col_stride=1,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
base = logits.clone()
|
||||
|
||||
# Warm up outside the capture: Triton compiles on first call, and autotune
|
||||
# launches would otherwise happen inside the graph.
|
||||
_apply_candidate_mask_strided(
|
||||
logits.clone(), row_ks, row_ke, candidates, BLOCK_SIZE, 1
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
_apply_candidate_mask_strided(logits, row_ks, row_ke, candidates, BLOCK_SIZE, 1)
|
||||
|
||||
for replay_ends in ([1, 131072, WIDE, 0], [WIDE, 1025, 3, 65536]):
|
||||
row_ke.copy_(torch.tensor(replay_ends, device="cuda", dtype=row_ke.dtype))
|
||||
logits.copy_(base)
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
ref = base.clone()
|
||||
apply_candidate_mask(ref, row_ks, row_ke, candidates, BLOCK_SIZE, 1)
|
||||
_assert_agrees_within_ends(ref, logits, row_ke, 1)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,262 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Regression test for causal masking in the AITER MLA verify MTP path.
|
||||
|
||||
When ``num_heads < 16`` the ROCm AITER MLA backend cannot use the ASM decode
|
||||
kernel for a multi-token block, so ``AiterMLAImpl.forward_mqa`` routes verify
|
||||
through ``mla_gluon``'s native 4-D MTP entry (``q`` shaped
|
||||
``[batch, qlen, nhead, dim]`` with ``use_2d_view=False``). aiter applies the
|
||||
causal tail internally: position ``t`` attends KV ``[0, seq_len - qlen + t]``.
|
||||
|
||||
The test drives the real builder and ``forward_mqa`` over a multi-token decode
|
||||
batch, intercepts the arguments actually handed to the Gluon kernel, and pins
|
||||
that the 4-D MTP contract is used with per-request ``block_table`` and
|
||||
``cache_seqlens`` rather than a flattened per-token paged-KV view.
|
||||
"""
|
||||
|
||||
import types
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm._aiter_ops import is_aiter_found
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
def _on_rocm_with_aiter() -> bool:
|
||||
return current_platform.is_rocm() and is_aiter_found()
|
||||
|
||||
|
||||
# The production verify MTP path is gfx950-only. This test replaces the Gluon
|
||||
# kernel with a spy and forces the architecture feature probe so the metadata
|
||||
# regression remains testable on every ROCm AITER CI runner.
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not _on_rocm_with_aiter(),
|
||||
reason="ROCM_AITER_MLA verify MTP requires ROCm and AITER",
|
||||
)
|
||||
|
||||
# Below the 16-head threshold that selects the MTP path. DeepSeek-V3 / R1 at
|
||||
# TP=16 and Kimi-K3 at TP=8 both land here.
|
||||
NUM_QUERY_HEADS = 12
|
||||
KV_LORA_RANK = 512
|
||||
QK_NOPE_HEAD_DIM = 128
|
||||
QK_ROPE_HEAD_DIM = 64
|
||||
V_HEAD_DIM = 128
|
||||
HEAD_SIZE = KV_LORA_RANK + QK_ROPE_HEAD_DIM
|
||||
|
||||
# 1 + num_speculative_tokens, i.e. the verify block length.
|
||||
QLEN = 8
|
||||
# Committed context per request, including a fresh request at zero context
|
||||
# where the whole KV range is the verify block itself.
|
||||
CONTEXT_LENS = [0, 1, 37, 512]
|
||||
# A cudagraph padding request: seq_len 0.
|
||||
PADDING_ROWS = 1
|
||||
|
||||
# The Gluon path flattens the KV cache to one page per token.
|
||||
PAGE_SIZE = 1
|
||||
MAX_MODEL_LEN = 1024
|
||||
MAX_NUM_SEQS = 8
|
||||
|
||||
# The builder resolves the verify routing and records it on the metadata, so
|
||||
# the gfx950 feature probe has to be forced for the build as well as for the
|
||||
# forward; ROCm CI also runs gfx942, where the real probe is False.
|
||||
_GLUON_SUPPORTED_TARGET = (
|
||||
"vllm.v1.attention.backends.mla.rocm_aiter_mla._gluon_mla_decode_supported"
|
||||
)
|
||||
|
||||
|
||||
def _seq_lens() -> list[int]:
|
||||
return [c + QLEN for c in CONTEXT_LENS] + [0] * PADDING_ROWS
|
||||
|
||||
|
||||
def _run_verify_block():
|
||||
"""Drive the real builder + forward_mqa over one multi-token verify block.
|
||||
|
||||
Returns ``(metadata, captured)`` where ``captured`` holds the Gluon kwargs.
|
||||
"""
|
||||
from tests.v1.attention.utils import (
|
||||
BatchSpec,
|
||||
create_common_attn_metadata,
|
||||
create_vllm_config,
|
||||
)
|
||||
from vllm.config import SpeculativeConfig
|
||||
from vllm.config.vllm import set_current_vllm_config
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAImpl
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.kv_cache_interface import MLAAttentionSpec
|
||||
from vllm.v1.worker.workspace import init_workspace_manager
|
||||
|
||||
device = torch.device("cuda:0")
|
||||
seq_lens = _seq_lens()
|
||||
num_reqs = len(seq_lens)
|
||||
# One flat page per token, arange block indices, plus room to spare.
|
||||
num_gpu_blocks = num_reqs * max(seq_lens) + 256
|
||||
|
||||
vllm_config = create_vllm_config(
|
||||
model_name="deepseek-ai/DeepSeek-R1",
|
||||
max_model_len=MAX_MODEL_LEN,
|
||||
num_gpu_blocks=num_gpu_blocks,
|
||||
block_size=PAGE_SIZE,
|
||||
max_num_seqs=MAX_NUM_SEQS,
|
||||
max_num_batched_tokens=8192,
|
||||
hf_config_override={"num_attention_heads": NUM_QUERY_HEADS},
|
||||
)
|
||||
# What raises reorder_batch_threshold to QLEN, so a QLEN-token block is
|
||||
# classified as a decode instead of a prefill. ngram needs no draft model.
|
||||
vllm_config.speculative_config = SpeculativeConfig(
|
||||
method="ngram", num_speculative_tokens=QLEN - 1
|
||||
)
|
||||
vllm_config.model_config.get_num_attention_heads = types.MethodType( # type: ignore[method-assign]
|
||||
lambda self, parallel_config, arch_config=None: NUM_QUERY_HEADS,
|
||||
vllm_config.model_config,
|
||||
)
|
||||
|
||||
spec = MLAAttentionSpec(
|
||||
block_size=PAGE_SIZE,
|
||||
num_kv_heads=1,
|
||||
head_size=vllm_config.model_config.get_head_size(),
|
||||
dtype=vllm_config.model_config.dtype,
|
||||
cache_dtype_str="auto",
|
||||
)
|
||||
|
||||
backend_cls = AttentionBackendEnum.ROCM_AITER_MLA.get_class()
|
||||
builder_cls = backend_cls.get_builder_cls()
|
||||
impl_cls = backend_cls.get_impl_cls()
|
||||
|
||||
# The builder reads layer.prefill_backend from static_forward_context; a
|
||||
# stub with the attribute is enough for metadata construction.
|
||||
layer_name = "placeholder"
|
||||
vllm_config.compilation_config.static_forward_context[layer_name] = (
|
||||
types.SimpleNamespace(
|
||||
prefill_backend=torch.empty((1,)),
|
||||
q_lora_rank=None,
|
||||
kv_lora_rank=KV_LORA_RANK,
|
||||
qk_nope_head_dim=QK_NOPE_HEAD_DIM,
|
||||
qk_rope_head_dim=QK_ROPE_HEAD_DIM,
|
||||
v_head_dim=V_HEAD_DIM,
|
||||
)
|
||||
)
|
||||
|
||||
init_workspace_manager(device)
|
||||
|
||||
batch_spec = BatchSpec(seq_lens=seq_lens, query_lens=[QLEN] * num_reqs)
|
||||
|
||||
captured: dict = {}
|
||||
|
||||
def spy(**kwargs):
|
||||
captured.update(kwargs)
|
||||
captured["q_nope"] = kwargs["q_nope"].detach().clone()
|
||||
captured["page_table"] = kwargs["page_table"].detach().clone()
|
||||
captured["seq_info"] = kwargs["seq_info"].detach().clone()
|
||||
|
||||
with (
|
||||
patch(_GLUON_SUPPORTED_TARGET, lambda: True),
|
||||
set_current_vllm_config(vllm_config),
|
||||
):
|
||||
builder = builder_cls(spec, [layer_name], vllm_config, device)
|
||||
common_attn_metadata = create_common_attn_metadata(
|
||||
batch_spec, PAGE_SIZE, device, arange_block_indices=True
|
||||
)
|
||||
metadata = builder.build(
|
||||
common_prefix_len=0, common_attn_metadata=common_attn_metadata
|
||||
)
|
||||
|
||||
impl = impl_cls(
|
||||
num_heads=NUM_QUERY_HEADS,
|
||||
head_size=HEAD_SIZE,
|
||||
scale=(QK_NOPE_HEAD_DIM + QK_ROPE_HEAD_DIM) ** -0.5,
|
||||
num_kv_heads=1,
|
||||
alibi_slopes=None,
|
||||
sliding_window=None,
|
||||
kv_cache_dtype="auto",
|
||||
logits_soft_cap=None,
|
||||
attn_type="decoder",
|
||||
kv_sharing_target_layer_name=None,
|
||||
q_lora_rank=None,
|
||||
kv_lora_rank=KV_LORA_RANK,
|
||||
qk_nope_head_dim=QK_NOPE_HEAD_DIM,
|
||||
qk_rope_head_dim=QK_ROPE_HEAD_DIM,
|
||||
qk_head_dim=QK_NOPE_HEAD_DIM + QK_ROPE_HEAD_DIM,
|
||||
v_head_dim=V_HEAD_DIM,
|
||||
kv_b_proj=None,
|
||||
)
|
||||
|
||||
num_rows = num_reqs * QLEN
|
||||
dtype = torch.bfloat16
|
||||
q_nope = torch.zeros(
|
||||
num_rows, NUM_QUERY_HEADS, KV_LORA_RANK, dtype=dtype, device=device
|
||||
)
|
||||
q_pe = torch.zeros(
|
||||
num_rows, NUM_QUERY_HEADS, QK_ROPE_HEAD_DIM, dtype=dtype, device=device
|
||||
)
|
||||
kv_cache = torch.zeros(
|
||||
num_gpu_blocks, PAGE_SIZE, HEAD_SIZE, dtype=dtype, device=device
|
||||
)
|
||||
|
||||
# The Gluon kernel only reads the metadata this test is about, so a spy in
|
||||
# its place keeps the assertions independent of the AITER build.
|
||||
with (
|
||||
patch(_GLUON_SUPPORTED_TARGET, lambda: True),
|
||||
patch(
|
||||
"vllm.v1.attention.backends.mla.rocm_aiter_mla._get_mla_gluon",
|
||||
lambda: spy,
|
||||
),
|
||||
):
|
||||
assert isinstance(impl, AiterMLAImpl)
|
||||
impl.forward_mqa((q_nope, q_pe), kv_cache, metadata, layer=None)
|
||||
|
||||
return metadata, captured
|
||||
|
||||
|
||||
def test_verify_mtp_uses_native_4d_gluon_entry():
|
||||
"""Verify uses 4-D q with per-request paged_kv metadata.
|
||||
|
||||
Regression guard: before the fix every flattened row of a request got that
|
||||
request's whole paged-KV range, so verify position t attended to the draft
|
||||
tokens after it. The native MTP path delegates causal masking to aiter.
|
||||
"""
|
||||
metadata, captured = _run_verify_block()
|
||||
|
||||
decode = metadata.decode
|
||||
assert decode is not None, "batch was not classified as a decode"
|
||||
assert decode.max_qo_len == QLEN, (
|
||||
f"expected a {QLEN}-token verify block, got max_qo_len="
|
||||
f"{decode.max_qo_len}; the MTP path under test was not reached"
|
||||
)
|
||||
assert decode.use_gluon_verify, (
|
||||
"the builder owns verify routing, so the metadata -- not the impl -- "
|
||||
"has to select the Gluon MTP entry for a small-head bf16 verify block"
|
||||
)
|
||||
assert captured, "forward_mqa did not reach the Gluon kernel"
|
||||
|
||||
seq_lens = _seq_lens()
|
||||
num_reqs = len(seq_lens)
|
||||
|
||||
q_nope = captured["q_nope"]
|
||||
assert q_nope.dim() == 4, (
|
||||
f"verify must pass 4-D q to mla_gluon, got {q_nope.dim()}-D shape "
|
||||
f"{tuple(q_nope.shape)}"
|
||||
)
|
||||
assert q_nope.shape[:2] == (num_reqs, QLEN), (
|
||||
f"expected q shape [num_reqs={num_reqs}, qlen={QLEN}, ...], "
|
||||
f"got {tuple(q_nope.shape)}"
|
||||
)
|
||||
assert q_nope.shape[2] == NUM_QUERY_HEADS
|
||||
|
||||
assert captured["use_2d_view"] is False
|
||||
page_table = captured["page_table"]
|
||||
seq_info = captured["seq_info"]
|
||||
assert page_table.dim() == 1, (
|
||||
f"verify MTP passes the 1-D paged_kv_indices buffer, got {page_table.dim()}-D"
|
||||
)
|
||||
assert seq_info.dim() == 1 and seq_info.numel() == num_reqs + 1, (
|
||||
f"verify MTP passes paged_kv_indptr [num_reqs + 1], got shape "
|
||||
f"{tuple(seq_info.shape)}"
|
||||
)
|
||||
assert decode is not None and decode.paged_kv_indptr is not None
|
||||
assert torch.equal(seq_info, decode.paged_kv_indptr[: num_reqs + 1])
|
||||
assert torch.equal(
|
||||
page_table[: int(seq_info[-1].item())],
|
||||
decode.paged_kv_indices[: int(seq_info[-1].item())],
|
||||
)
|
||||
@@ -1,225 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""ROCm kernel correctness tests for AITER MLA decode.
|
||||
|
||||
Compares ``rocm_aiter_ops.mla_decode_fwd`` against a pure PyTorch reference
|
||||
under the absorbed MLA formulation (DeepSeek-V3/V4) with varied batch sizes,
|
||||
head counts, and sequence lengths.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
_SKIP_NON_MI3XX = True
|
||||
if current_platform.is_rocm():
|
||||
from vllm.platforms.rocm import on_mi3xx
|
||||
|
||||
_SKIP_NON_MI3XX = not on_mi3xx()
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-specific tests"),
|
||||
pytest.mark.skipif(_SKIP_NON_MI3XX, reason="MI300/MI350 ROCm only"),
|
||||
]
|
||||
|
||||
# DeepSeek-V3/V4 MLA config.
|
||||
Q_HEAD_DIM = 576 # kv_lora_rank + qk_rope_head_dim
|
||||
V_HEAD_DIM = 512 # kv_lora_rank
|
||||
SM_SCALE = Q_HEAD_DIM**-0.5
|
||||
|
||||
NUM_HEADS = [16, 128]
|
||||
BATCH_SIZES = [1, 4, 16]
|
||||
KV_SEQ_LENS = [16, 256]
|
||||
|
||||
ATOL, RTOL = 1.5e-2, 1e-2
|
||||
|
||||
|
||||
def _require_aiter():
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported
|
||||
|
||||
if not is_aiter_found_and_supported():
|
||||
pytest.skip("aiter is required on supported ROCm hardware for this test")
|
||||
|
||||
|
||||
def _ref_mla_decode(
|
||||
q: torch.Tensor,
|
||||
kv_buffer: torch.Tensor,
|
||||
kv_indptr: torch.Tensor,
|
||||
kv_indices: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Pure PyTorch reference for absorbed MLA decode.
|
||||
|
||||
K uses full q_head_dim (576) for scoring, V uses first v_head_dim (512)
|
||||
for output. This mirrors the production kernel's absorbed formulation.
|
||||
"""
|
||||
batch_size, num_heads, _ = q.shape
|
||||
output = torch.zeros(
|
||||
batch_size, num_heads, V_HEAD_DIM, dtype=q.dtype, device=q.device
|
||||
)
|
||||
|
||||
for b in range(batch_size):
|
||||
start = kv_indptr[b].item()
|
||||
end = kv_indptr[b + 1].item()
|
||||
token_indices = kv_indices[start:end]
|
||||
|
||||
k = kv_buffer[token_indices].float()
|
||||
v = kv_buffer[token_indices, :V_HEAD_DIM].float()
|
||||
|
||||
for h in range(num_heads):
|
||||
q_h = q[b, h, :].float()
|
||||
scores = torch.mv(k, q_h) * SM_SCALE
|
||||
weights = torch.softmax(scores, dim=0)
|
||||
output[b, h, :] = torch.mv(v.t(), weights).to(q.dtype)
|
||||
|
||||
return output
|
||||
|
||||
|
||||
def _make_inputs(
|
||||
batch_size: int,
|
||||
nhead: int,
|
||||
kv_seq_len: int,
|
||||
*,
|
||||
contiguous_indices: bool = True,
|
||||
):
|
||||
"""Build valid MLA decode inputs on the current CUDA device.
|
||||
|
||||
When contiguous_indices=False, tokens are randomly scattered in a 2x pool
|
||||
to simulate real paged allocation.
|
||||
"""
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAHelper
|
||||
|
||||
actual_nhead = AiterMLAHelper.get_actual_mla_num_heads(nhead)
|
||||
total_kv_tokens = batch_size * kv_seq_len
|
||||
|
||||
pool_size = total_kv_tokens if contiguous_indices else total_kv_tokens * 2
|
||||
|
||||
q = torch.randn(
|
||||
batch_size, actual_nhead, Q_HEAD_DIM, dtype=torch.bfloat16, device="cuda"
|
||||
)
|
||||
kv_buffer = torch.randn(
|
||||
pool_size, 1, Q_HEAD_DIM, dtype=torch.bfloat16, device="cuda"
|
||||
)
|
||||
o = torch.zeros(
|
||||
batch_size, actual_nhead, V_HEAD_DIM, dtype=torch.bfloat16, device="cuda"
|
||||
)
|
||||
|
||||
qo_indptr = torch.arange(0, batch_size + 1, dtype=torch.int32, device="cuda")
|
||||
kv_indptr = (
|
||||
torch.arange(0, batch_size + 1, dtype=torch.int32, device="cuda") * kv_seq_len
|
||||
)
|
||||
kv_last_page_lens = torch.ones(batch_size, dtype=torch.int32, device="cuda")
|
||||
|
||||
if contiguous_indices:
|
||||
kv_indices = torch.arange(0, total_kv_tokens, dtype=torch.int32, device="cuda")
|
||||
else:
|
||||
perm = torch.randperm(pool_size, device="cuda")[:total_kv_tokens]
|
||||
kv_indices = perm.to(torch.int32)
|
||||
|
||||
return {
|
||||
"q": q,
|
||||
"kv_buffer": kv_buffer,
|
||||
"o": o,
|
||||
"qo_indptr": qo_indptr,
|
||||
"kv_indptr": kv_indptr,
|
||||
"kv_indices": kv_indices,
|
||||
"kv_last_page_lens": kv_last_page_lens,
|
||||
"actual_nhead": actual_nhead,
|
||||
}
|
||||
|
||||
|
||||
def _run_kernel(inputs: dict) -> torch.Tensor:
|
||||
"""Run decode through the production rocm_aiter_ops path."""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
rocm_aiter_ops.mla_decode_fwd(
|
||||
inputs["q"],
|
||||
inputs["kv_buffer"],
|
||||
inputs["o"],
|
||||
SM_SCALE,
|
||||
inputs["qo_indptr"],
|
||||
1,
|
||||
inputs["kv_indptr"],
|
||||
inputs["kv_indices"],
|
||||
inputs["kv_last_page_lens"],
|
||||
)
|
||||
return inputs["o"]
|
||||
|
||||
|
||||
def _ref_output(inputs: dict) -> torch.Tensor:
|
||||
"""Compute reference output from the same inputs."""
|
||||
kv_flat = inputs["kv_buffer"].squeeze(1)
|
||||
return _ref_mla_decode(
|
||||
inputs["q"], kv_flat, inputs["kv_indptr"], inputs["kv_indices"]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("nhead", NUM_HEADS)
|
||||
@pytest.mark.parametrize("batch_size", BATCH_SIZES)
|
||||
@pytest.mark.parametrize("kv_seq_len", KV_SEQ_LENS)
|
||||
@torch.inference_mode()
|
||||
def test_mla_decode_accuracy(
|
||||
nhead: int,
|
||||
batch_size: int,
|
||||
kv_seq_len: int,
|
||||
) -> None:
|
||||
"""BF16 decode accuracy vs PyTorch reference (contiguous indices)."""
|
||||
_require_aiter()
|
||||
set_random_seed(0)
|
||||
|
||||
inputs = _make_inputs(batch_size, nhead, kv_seq_len)
|
||||
output = _run_kernel(inputs)
|
||||
output_ref = _ref_output(inputs)
|
||||
|
||||
torch.testing.assert_close(output, output_ref, atol=ATOL, rtol=RTOL)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("nhead", NUM_HEADS)
|
||||
@pytest.mark.parametrize("batch_size", [1, 4, 8])
|
||||
@pytest.mark.parametrize("kv_seq_len", [16, 128])
|
||||
@torch.inference_mode()
|
||||
def test_mla_decode_noncontiguous_indices(
|
||||
nhead: int,
|
||||
batch_size: int,
|
||||
kv_seq_len: int,
|
||||
) -> None:
|
||||
"""BF16 decode accuracy with shuffled kv_indices (paged allocation)."""
|
||||
_require_aiter()
|
||||
set_random_seed(0)
|
||||
|
||||
inputs = _make_inputs(batch_size, nhead, kv_seq_len, contiguous_indices=False)
|
||||
output = _run_kernel(inputs)
|
||||
output_ref = _ref_output(inputs)
|
||||
|
||||
torch.testing.assert_close(output, output_ref, atol=ATOL, rtol=RTOL)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_mla_decode_determinism() -> None:
|
||||
"""Repeated decode calls produce bitwise-identical results."""
|
||||
_require_aiter()
|
||||
set_random_seed(0)
|
||||
|
||||
inputs = _make_inputs(batch_size=4, nhead=128, kv_seq_len=16)
|
||||
out_first = _run_kernel(inputs).clone()
|
||||
|
||||
for _ in range(3):
|
||||
inputs["o"] = torch.zeros_like(inputs["o"])
|
||||
out = _run_kernel(inputs)
|
||||
torch.testing.assert_close(out, out_first, atol=0, rtol=0)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_mla_decode_smoke() -> None:
|
||||
"""Basic sanity: correct shape, dtype, finite, non-zero."""
|
||||
_require_aiter()
|
||||
set_random_seed(0)
|
||||
|
||||
inputs = _make_inputs(batch_size=4, nhead=128, kv_seq_len=64)
|
||||
output = _run_kernel(inputs)
|
||||
|
||||
assert output.shape == (4, 128, V_HEAD_DIM)
|
||||
assert output.dtype == torch.bfloat16
|
||||
assert torch.isfinite(output).all()
|
||||
assert not torch.all(output == 0)
|
||||
@@ -1,213 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Regression test for AITER MLA persistent decode metadata dtypes.
|
||||
|
||||
For the gfx950 fp8/fp8 nhead=32 qlen=1 fold path, the split/reduce metadata
|
||||
layout depends on the q/kv element size. The builder must forward dtype_q/dtype_kv
|
||||
to ``get_mla_metadata_v1``; omitting them lays out the work for the wrong dtype
|
||||
and corrupts decode output. The test pins the builder's metadata to a golden
|
||||
recomputed at runtime with the explicit correct dtypes.
|
||||
"""
|
||||
|
||||
import types
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm._aiter_ops import is_aiter_found
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
def _on_gfx950() -> bool:
|
||||
if not (current_platform.is_rocm() and is_aiter_found()):
|
||||
return False
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
|
||||
return on_gfx950()
|
||||
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not _on_gfx950(),
|
||||
reason="AITER MLA fp8 persistent decode metadata is gfx950-only",
|
||||
)
|
||||
|
||||
# The fold path that the bug corrupted: fp8 query + fp8 KV-cache, 32 query
|
||||
# heads, single-token decode, batch 128, context 8192, page_size 1.
|
||||
NUM_QUERY_HEADS = 32
|
||||
DECODE_QLEN = 1
|
||||
BATCH_SIZE = 128
|
||||
CONTEXT_LEN = 8192
|
||||
PAGE_SIZE = 1
|
||||
|
||||
# On the fp8 KV-cache path the builder forwards the platform fp8 dtype (aiter
|
||||
# dtypes.fp8) for both q and kv. Mirror it via current_platform.fp8_dtype()
|
||||
# instead of hardcoding a literal (see #47276).
|
||||
EXPECTED_Q_DTYPE = current_platform.fp8_dtype()
|
||||
EXPECTED_KV_DTYPE = current_platform.fp8_dtype()
|
||||
|
||||
# The split/reduce content tensors filled by get_mla_metadata_v1. work_meta_data
|
||||
# is excluded: it holds raw device pointers, never equal across allocations.
|
||||
_CONTENT_METADATA_FIELDS = (
|
||||
"work_indptr",
|
||||
"work_info_set",
|
||||
"reduce_indptr",
|
||||
"reduce_final_map",
|
||||
"reduce_partial_map",
|
||||
)
|
||||
|
||||
# The builder's get_mla_metadata_v1 call passes 6 input args then 6 output
|
||||
# buffers (see AiterMLAMetadataBuilder._build_decode). Output order -> field.
|
||||
_NUM_INPUT_ARGS = 6
|
||||
_OUTPUT_ARG_FIELDS = (
|
||||
"work_meta_data",
|
||||
"work_info_set",
|
||||
"work_indptr",
|
||||
"reduce_indptr",
|
||||
"reduce_final_map",
|
||||
"reduce_partial_map",
|
||||
)
|
||||
|
||||
|
||||
def _build_decode_metadata():
|
||||
"""Build AITER MLA decode metadata for the fp8/fp8 nhead=32 fold path.
|
||||
|
||||
Returns ``(metadata, captured)`` where ``captured`` records the positional
|
||||
args/kwargs the builder passed to ``get_mla_metadata_v1``, so the golden can
|
||||
be recomputed from the identical inputs.
|
||||
"""
|
||||
from tests.v1.attention.utils import (
|
||||
BatchSpec,
|
||||
create_common_attn_metadata,
|
||||
create_vllm_config,
|
||||
)
|
||||
from vllm.config.vllm import set_current_vllm_config
|
||||
from vllm.model_executor.layers.attention.mla_attention import get_mla_dims
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.kv_cache_interface import MLAAttentionSpec
|
||||
from vllm.v1.worker.workspace import init_workspace_manager
|
||||
|
||||
device = torch.device("cuda:0")
|
||||
|
||||
vllm_config = create_vllm_config(
|
||||
model_name="deepseek-ai/DeepSeek-R1",
|
||||
max_model_len=CONTEXT_LEN,
|
||||
# One flat page per token (page_size=1); +buffer for the null block.
|
||||
num_gpu_blocks=BATCH_SIZE * CONTEXT_LEN + 200,
|
||||
block_size=PAGE_SIZE,
|
||||
max_num_seqs=BATCH_SIZE,
|
||||
max_num_batched_tokens=8192,
|
||||
hf_config_override={"num_attention_heads": NUM_QUERY_HEADS},
|
||||
)
|
||||
vllm_config.cache_config.cache_dtype = "fp8"
|
||||
|
||||
spec = MLAAttentionSpec(
|
||||
block_size=PAGE_SIZE,
|
||||
num_kv_heads=1,
|
||||
head_size=vllm_config.model_config.get_head_size(),
|
||||
dtype=vllm_config.model_config.dtype,
|
||||
cache_dtype_str="fp8",
|
||||
)
|
||||
|
||||
builder_cls = AttentionBackendEnum.ROCM_AITER_MLA.get_class().get_builder_cls()
|
||||
|
||||
# The builder reads prefill_backend and the MLA latent dims off the layer,
|
||||
# so the stub carries both; the dims come from the model config to stay in
|
||||
# step with the checkpoint.
|
||||
mla_dims = get_mla_dims(vllm_config.model_config)
|
||||
layer_name = "placeholder"
|
||||
vllm_config.compilation_config.static_forward_context[layer_name] = (
|
||||
types.SimpleNamespace(
|
||||
prefill_backend=torch.empty((1,)),
|
||||
q_lora_rank=mla_dims.q_lora_rank,
|
||||
kv_lora_rank=mla_dims.kv_lora_rank,
|
||||
qk_nope_head_dim=mla_dims.qk_nope_head_dim,
|
||||
qk_rope_head_dim=mla_dims.qk_rope_head_dim,
|
||||
v_head_dim=mla_dims.v_head_dim,
|
||||
)
|
||||
)
|
||||
|
||||
init_workspace_manager(device)
|
||||
|
||||
batch_spec = BatchSpec(
|
||||
seq_lens=[CONTEXT_LEN] * BATCH_SIZE,
|
||||
query_lens=[DECODE_QLEN] * BATCH_SIZE,
|
||||
)
|
||||
|
||||
captured: dict = {}
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
builder = builder_cls(spec, [layer_name], vllm_config, device)
|
||||
common_attn_metadata = create_common_attn_metadata(
|
||||
batch_spec, PAGE_SIZE, device, arange_block_indices=True
|
||||
)
|
||||
|
||||
import aiter
|
||||
|
||||
real_get_mla_metadata_v1 = aiter.get_mla_metadata_v1
|
||||
|
||||
def spy(*args, **kwargs):
|
||||
captured["args"] = args
|
||||
captured["kwargs"] = dict(kwargs)
|
||||
return real_get_mla_metadata_v1(*args, **kwargs)
|
||||
|
||||
with patch("aiter.get_mla_metadata_v1", spy):
|
||||
metadata = builder.build(
|
||||
common_prefix_len=0,
|
||||
common_attn_metadata=common_attn_metadata,
|
||||
)
|
||||
|
||||
return metadata, captured
|
||||
|
||||
|
||||
def _compute_golden_metadata(captured: dict) -> dict[str, torch.Tensor]:
|
||||
"""Recompute the persistent metadata with explicit fp8/bf16 dtypes.
|
||||
|
||||
Replays ``get_mla_metadata_v1`` on the builder's exact input tensors with
|
||||
fresh output buffers and the explicitly-correct dtypes. This reference must
|
||||
match the builder's output when the fix is in place.
|
||||
"""
|
||||
import aiter
|
||||
|
||||
args = captured["args"]
|
||||
inputs = args[:_NUM_INPUT_ARGS]
|
||||
# Fresh copies so the golden does not alias the builder's persistent buffers.
|
||||
fresh_outputs = [arg.clone() for arg in args[_NUM_INPUT_ARGS:]]
|
||||
|
||||
golden_kwargs = dict(captured["kwargs"])
|
||||
golden_kwargs["dtype_q"] = EXPECTED_Q_DTYPE
|
||||
golden_kwargs["dtype_kv"] = EXPECTED_KV_DTYPE
|
||||
|
||||
aiter.get_mla_metadata_v1(*inputs, *fresh_outputs, **golden_kwargs)
|
||||
|
||||
return dict(zip(_OUTPUT_ARG_FIELDS, fresh_outputs))
|
||||
|
||||
|
||||
def test_persistent_decode_metadata_matches_fp8_golden():
|
||||
"""The builder's metadata must match the dtype-correct golden.
|
||||
|
||||
Regression guard: the fixed builder forwards fp8/bf16 dtypes so its
|
||||
split/reduce metadata matches the golden recomputed with those explicit
|
||||
dtypes. Dropping the dtypes (the original bug) produces a different layout
|
||||
and fails this test.
|
||||
"""
|
||||
metadata, captured = _build_decode_metadata()
|
||||
|
||||
# qlen=1 must take the persistent-metadata path for this to be meaningful.
|
||||
assert metadata.decode is not None
|
||||
assert metadata.decode.has_persistent_metadata
|
||||
assert metadata.work_meta_data is not None
|
||||
|
||||
golden = _compute_golden_metadata(captured)
|
||||
|
||||
mismatched = [
|
||||
name
|
||||
for name in _CONTENT_METADATA_FIELDS
|
||||
if getattr(metadata, name).shape != golden[name].shape
|
||||
or not torch.equal(getattr(metadata, name), golden[name])
|
||||
]
|
||||
assert not mismatched, (
|
||||
"AITER MLA persistent decode metadata does not match the fp8/bf16 "
|
||||
f"golden for fields {mismatched}; the builder must forward "
|
||||
"dtype_q/dtype_kv to get_mla_metadata_v1."
|
||||
)
|
||||
@@ -1,246 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Accuracy test for the ROCm AITER FP8 MLA prefill path.
|
||||
|
||||
The FP8 MLA prefill path
|
||||
(``AiterMLAImpl._mla_fp8_prefill_attn`` -> ``mla_prefill_ps_asm_fwd`` +
|
||||
``mla_reduce_v1``) is auto-enabled on gfx950 (MI355) when AITER ships those
|
||||
kernels, and it has no other test coverage. This test drives the *real*
|
||||
metadata builder and impl methods (via ``object.__new__``, mirroring
|
||||
``test_rocm_aiter_mla_head_padding.py::test_h12_aiter_mla_decode_matches_reference``)
|
||||
so it exercises the actual persistent-scheduling metadata contract
|
||||
(``get_ps_metadata_v1``) plus both kernels -- rather than re-implementing them --
|
||||
and compares the output against a causal SDPA reference.
|
||||
|
||||
gfx950-only: ``mla_prefill_ps_asm_fwd`` has no gfx942 build, so this is skipped
|
||||
on every other platform.
|
||||
"""
|
||||
|
||||
import math
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
def _fp8_prefill_available() -> bool:
|
||||
if not (current_platform.is_rocm() and torch.cuda.is_available()):
|
||||
return False
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla import (
|
||||
_fp8_mla_prefill_supported,
|
||||
)
|
||||
|
||||
return _fp8_mla_prefill_supported()
|
||||
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not _fp8_prefill_available(),
|
||||
reason="FP8 MLA prefill needs gfx950 (MI355) + an AITER build exporting "
|
||||
"mla_prefill_ps_asm_fwd / mla_reduce_v1",
|
||||
)
|
||||
|
||||
# DeepSeek-style MLA head dims after kv_b_proj decompression (q/k carry the
|
||||
# nope+rope dims, v carries v_head_dim).
|
||||
#
|
||||
# The kernel requires 16-aligned heads, so _mla_fp8_prefill_attn replicate-pads
|
||||
# q/k/v up to get_fp8_prefill_num_heads(num_heads) and slices the output back.
|
||||
# Cover all four regimes:
|
||||
# 8 -> padded to 16 (divisor, repeat_interleave), so the real heads sit at
|
||||
# stride 2 in the padded output. Live case: Kimi-K2.x at TP8.
|
||||
# 12 -> padded to 16 (non-divisor, tile+slice). Live case: a K3 rank at TP8.
|
||||
# 16 -> no padding, output aliases the caller's buffer.
|
||||
# 24 -> padded to 32. No current model and TP reaches this band (96 heads
|
||||
# would need TP4, which does not fit in 288 GiB; 128-head models give
|
||||
# 128/64/32/16/8), so this case exists to keep the general path honest
|
||||
# for future architectures: it is the only one where padding above
|
||||
# 16 actually happens, so it is what proves the replicate-pad/slice-back
|
||||
# argument holds for a target other than 16.
|
||||
HEAD_COUNTS = [8, 12, 16, 24]
|
||||
QK_NOPE_HEAD_DIM = 128
|
||||
QK_ROPE_HEAD_DIM = 64
|
||||
QK_HEAD_DIM = QK_NOPE_HEAD_DIM + QK_ROPE_HEAD_DIM # 192
|
||||
V_HEAD_DIM = 128
|
||||
SCALE = 1.0 / math.sqrt(QK_HEAD_DIM)
|
||||
|
||||
# Dominated by fp8 (e4m3) rounding of q/k/v. Measured on MI355: worst-element
|
||||
# abs error ~0.054, so atol=0.1 is ~2x that noise floor; rtol=0.05 bounds larger
|
||||
# elements.
|
||||
ATOL, RTOL = 1e-1, 5e-2
|
||||
|
||||
# The assembly kernel writes bf16 through a raw output pointer. Keep its tensors
|
||||
# and workspace reservation on one dtype.
|
||||
ATTN_OUT_DTYPE = torch.bfloat16
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _workspace_manager():
|
||||
"""The FP8 prefill path draws per-call scratch from the global workspace
|
||||
manager, which ``GPUModelRunner.__init__`` normally initializes. Set it up
|
||||
for the test and tear it down so no global state leaks to other tests.
|
||||
"""
|
||||
from vllm.v1.worker.workspace import (
|
||||
init_workspace_manager,
|
||||
reset_workspace_manager,
|
||||
)
|
||||
|
||||
init_workspace_manager(torch.device("cuda"))
|
||||
yield
|
||||
reset_workspace_manager()
|
||||
|
||||
|
||||
def _make_impl(num_heads: int):
|
||||
"""Minimal AiterMLAImpl exposing only what _mla_fp8_prefill_attn reads."""
|
||||
from aiter import mla_prefill_ps_asm_fwd, mla_reduce_v1
|
||||
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAImpl
|
||||
|
||||
impl = object.__new__(AiterMLAImpl)
|
||||
impl.num_heads = num_heads
|
||||
impl.v_head_dim = V_HEAD_DIM
|
||||
impl.scale = SCALE
|
||||
impl._mla_prefill_ps_asm_fwd = mla_prefill_ps_asm_fwd
|
||||
impl._mla_reduce_v1 = mla_reduce_v1
|
||||
return impl
|
||||
|
||||
|
||||
def _build_prefill_metadata(
|
||||
seq_lens: list[int],
|
||||
device: torch.device,
|
||||
num_heads: int,
|
||||
attn_out_dtype: torch.dtype = ATTN_OUT_DTYPE,
|
||||
):
|
||||
"""Build metadata with the production builder and output dtype."""
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAMetadataBuilder
|
||||
|
||||
qo_indptr_cpu = torch.zeros(len(seq_lens) + 1, dtype=torch.int32)
|
||||
qo_indptr_cpu[1:] = torch.tensor(seq_lens, dtype=torch.int32).cumsum(0)
|
||||
qo_indptr = qo_indptr_cpu.to(device)
|
||||
total_q = int(qo_indptr_cpu[-1].item())
|
||||
max_q = max(seq_lens)
|
||||
|
||||
builder = object.__new__(AiterMLAMetadataBuilder)
|
||||
builder.num_heads = num_heads
|
||||
builder.mla_dims = SimpleNamespace(v_head_dim=V_HEAD_DIM)
|
||||
builder._init_fp8_prefill_ps_buffers(
|
||||
max_num_reqs=len(seq_lens),
|
||||
max_prefill_qlen=max_q,
|
||||
max_num_batched_tokens=total_q,
|
||||
attn_out_dtype=attn_out_dtype,
|
||||
device=device,
|
||||
)
|
||||
|
||||
# SimpleNamespace stands in for the metadata dataclasses: the builder only
|
||||
# reads prefill.{query_start_loc,max_query_len}, num_decodes, and
|
||||
# common.query_start_loc_cpu, then populates metadata.fp8_prefill_*.
|
||||
prefill = SimpleNamespace(query_start_loc=qo_indptr, max_query_len=max_q)
|
||||
metadata = SimpleNamespace(prefill=prefill, num_decodes=0)
|
||||
common = SimpleNamespace(query_start_loc_cpu=qo_indptr_cpu)
|
||||
builder._build_fp8_prefill_ps_metadata(metadata, common)
|
||||
return metadata, total_q, builder
|
||||
|
||||
|
||||
def _reference(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor) -> torch.Tensor:
|
||||
"""Causal SDPA reference on fp8-quantized inputs (matches kernel dtype)."""
|
||||
fp8 = current_platform.fp8_dtype()
|
||||
qf = q.to(fp8).float().transpose(0, 1) # [H, S, Dqk]
|
||||
kf = k.to(fp8).float().transpose(0, 1)
|
||||
vf = v.to(fp8).float().transpose(0, 1) # [H, S, Dv]
|
||||
out = F.scaled_dot_product_attention(qf, kf, vf, is_causal=True, scale=SCALE)
|
||||
return out.transpose(0, 1) # [S, H, Dv]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_heads", HEAD_COUNTS)
|
||||
@pytest.mark.parametrize("seq_len", [128, 512])
|
||||
@torch.inference_mode()
|
||||
def test_fp8_prefill_matches_reference(seq_len: int, num_heads: int) -> None:
|
||||
device = torch.device("cuda")
|
||||
torch.manual_seed(0)
|
||||
|
||||
metadata, total_q, _ = _build_prefill_metadata([seq_len], device, num_heads)
|
||||
|
||||
# Lock the workspace, as GPUModelRunner does after warmup, so the forward
|
||||
# cannot quietly grow it. The builder reserves scratch at num_head_k and
|
||||
# the forward requests it at its own padded width; if a future change makes
|
||||
# the forward's width exceed the builder's, that is an under-reservation
|
||||
# which is invisible on an unlocked workspace (it just grows) and fatal in
|
||||
# a real serve. Locking turns it into a failure here.
|
||||
#
|
||||
# Note this does not catch the reverse -- a builder wider than the forward
|
||||
# merely over-reserves -- so it would not have flagged the pre-PR
|
||||
# max(16, num_heads) sizing, which over-reserves 2.8x at 24 heads.
|
||||
from vllm.v1.worker.workspace import lock_workspace
|
||||
|
||||
lock_workspace()
|
||||
|
||||
qkv_kwargs = dict(dtype=ATTN_OUT_DTYPE, device=device)
|
||||
q = torch.randn(total_q, num_heads, QK_HEAD_DIM, **qkv_kwargs)
|
||||
k = torch.randn(total_q, num_heads, QK_HEAD_DIM, **qkv_kwargs)
|
||||
v = torch.randn(total_q, num_heads, V_HEAD_DIM, **qkv_kwargs)
|
||||
out = torch.zeros(
|
||||
total_q, num_heads * V_HEAD_DIM, dtype=ATTN_OUT_DTYPE, device=device
|
||||
)
|
||||
|
||||
impl = _make_impl(num_heads)
|
||||
impl._mla_fp8_prefill_attn(q, k, v, metadata, out)
|
||||
|
||||
out_ref = _reference(q, k, v)
|
||||
|
||||
assert torch.isfinite(out).all()
|
||||
torch.testing.assert_close(
|
||||
out.view(total_q, num_heads, V_HEAD_DIM).float(),
|
||||
out_ref.float(),
|
||||
atol=ATOL,
|
||||
rtol=RTOL,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_heads", HEAD_COUNTS)
|
||||
def test_fp8_prefill_metadata_width_matches_forward(num_heads: int) -> None:
|
||||
"""The PS metadata must be built for the head count the kernel is handed.
|
||||
|
||||
``_init_fp8_prefill_ps_buffers``/``build()`` size the work and reduce maps
|
||||
from ``num_head_k``, while ``_mla_fp8_prefill_attn`` pads q/k/v to its own
|
||||
target. Divergence is not a numerical bug in either direction -- maps built
|
||||
narrower than the kernel width still cover the real heads, since padding
|
||||
only ever appends duplicate heads above them. It is a sizing bug: narrower
|
||||
maps mean a lower head alignment, hence more partial tiles and a larger
|
||||
reservation, while wider ones under-reserve the forward's scratch. Pin the
|
||||
two to one helper rather than to independent expressions.
|
||||
"""
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAHelper
|
||||
|
||||
width = AiterMLAHelper.get_fp8_prefill_num_heads(num_heads)
|
||||
assert width % AiterMLAHelper._AITER_MIN_MLA_HEADS == 0
|
||||
assert width >= num_heads
|
||||
assert width - num_heads < AiterMLAHelper._AITER_MIN_MLA_HEADS
|
||||
# Every count the enable gate accepts must have a usable padded width.
|
||||
assert AiterMLAHelper.is_valid_num_heads(num_heads)
|
||||
|
||||
# The builder must actually size its maps at that width. Assert against the
|
||||
# buffer it allocated rather than recomputing the width here, or the test
|
||||
# passes no matter what _init_fp8_prefill_ps_buffers does.
|
||||
#
|
||||
# fp8_ps_reduce_partial_map is sized by the partial-tile count, which is
|
||||
# gcd-driven and so is a sharp probe of the width used: a lower head
|
||||
# alignment yields *more* tiles. At 24 heads, sizing at the unpadded count
|
||||
# gives 4x the tiles (64 vs 16) and ~2.8x the scratch reservation (193.6
|
||||
# vs 68.6 MiB at batch=1/qlen=512) -- not wrong numerically (the 24-wide
|
||||
# maps still cover the real heads) but the reason to pin the widths
|
||||
# together.
|
||||
from aiter import get_ps_metadata_info_v1
|
||||
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla import _FP8_PREFILL_TILE_Q
|
||||
|
||||
def _tiles(nhk: int) -> int:
|
||||
return get_ps_metadata_info_v1(
|
||||
batch_size=1,
|
||||
num_head_k=nhk,
|
||||
max_qlen=512,
|
||||
qlen_granularity=_FP8_PREFILL_TILE_Q,
|
||||
)[5][0]
|
||||
|
||||
_, _, builder = _build_prefill_metadata([512], torch.device("cuda"), num_heads)
|
||||
assert builder.fp8_ps_reduce_partial_map.numel() == _tiles(width)
|
||||
@@ -1,252 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for ROCm AITER MLA FP8 support detection."""
|
||||
|
||||
import sys
|
||||
import types
|
||||
from typing import Any
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
_SKIP_UNSUPPORTED_AITER_HARDWARE = True
|
||||
if current_platform.is_rocm():
|
||||
from vllm.platforms.rocm import get_cdna_version
|
||||
|
||||
_SKIP_UNSUPPORTED_AITER_HARDWARE = get_cdna_version() <= 2
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_rocm(), reason="ROCm-specific tests"
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def reset_aiter_mla_support_cache(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
import vllm._aiter_ops as aiter_ops
|
||||
|
||||
monkeypatch.setattr(aiter_ops, "_AITER_MLA_SUPPORTS_FP8", None)
|
||||
|
||||
|
||||
def _install_fake_aiter_modules(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
*,
|
||||
supports_fp8: bool,
|
||||
supports_causal: bool = False,
|
||||
) -> None:
|
||||
aiter_mod: Any = types.ModuleType("aiter")
|
||||
mla_mod: Any = types.ModuleType("aiter.mla")
|
||||
|
||||
if supports_fp8:
|
||||
|
||||
def mla_decode_fwd_with_fp8(
|
||||
q,
|
||||
kv_buffer,
|
||||
kv_indptr,
|
||||
kv_indices,
|
||||
o,
|
||||
sm_scale,
|
||||
q_scale=None,
|
||||
kv_scale=None,
|
||||
):
|
||||
return None
|
||||
|
||||
mla_decode_fwd: Any = mla_decode_fwd_with_fp8
|
||||
|
||||
else:
|
||||
|
||||
def mla_decode_fwd_without_fp8(
|
||||
q,
|
||||
kv_buffer,
|
||||
kv_indptr,
|
||||
kv_indices,
|
||||
o,
|
||||
sm_scale,
|
||||
):
|
||||
return None
|
||||
|
||||
mla_decode_fwd = mla_decode_fwd_without_fp8
|
||||
|
||||
if supports_causal:
|
||||
inner = mla_decode_fwd
|
||||
|
||||
def mla_decode_fwd_with_causal(*args, causal=True, **kwargs):
|
||||
return inner(*args, **kwargs)
|
||||
|
||||
mla_decode_fwd = mla_decode_fwd_with_causal
|
||||
|
||||
mla_mod.mla_decode_fwd = mla_decode_fwd
|
||||
aiter_mod.mla = mla_mod
|
||||
|
||||
monkeypatch.setitem(sys.modules, "aiter", aiter_mod)
|
||||
monkeypatch.setitem(sys.modules, "aiter.mla", mla_mod)
|
||||
|
||||
|
||||
def test_aiter_mla_fp8_support_detects_fp8_signature(monkeypatch):
|
||||
"""The support check should detect q_scale and kv_scale parameters."""
|
||||
from vllm._aiter_ops import _check_aiter_mla_fp8_support
|
||||
|
||||
_install_fake_aiter_modules(monkeypatch, supports_fp8=True)
|
||||
|
||||
assert _check_aiter_mla_fp8_support() is True
|
||||
|
||||
|
||||
def test_aiter_mla_fp8_support_rejects_missing_fp8_signature(monkeypatch):
|
||||
"""The support check should return False when FP8 params are absent."""
|
||||
from vllm._aiter_ops import _check_aiter_mla_fp8_support
|
||||
|
||||
_install_fake_aiter_modules(monkeypatch, supports_fp8=False)
|
||||
|
||||
assert _check_aiter_mla_fp8_support() is False
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
_SKIP_UNSUPPORTED_AITER_HARDWARE,
|
||||
reason="Installed AITER MLA FP8 check requires CDNA 3 or newer",
|
||||
)
|
||||
def test_installed_aiter_mla_supports_fp8():
|
||||
"""Supported ROCm CI must provide AITER with MLA FP8 scaling."""
|
||||
from vllm._aiter_ops import (
|
||||
_check_aiter_mla_fp8_support,
|
||||
is_aiter_found_and_supported,
|
||||
)
|
||||
|
||||
assert is_aiter_found_and_supported(), (
|
||||
"AITER must be installed on supported ROCm hardware"
|
||||
)
|
||||
assert _check_aiter_mla_fp8_support() is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"error_type",
|
||||
[ImportError, ModuleNotFoundError, AttributeError, ValueError, TypeError],
|
||||
)
|
||||
def test_aiter_mla_fp8_support_handles_signature_errors(monkeypatch, error_type):
|
||||
"""The support check should fail closed on import or signature problems."""
|
||||
import vllm._aiter_ops as aiter_ops
|
||||
from vllm._aiter_ops import _check_aiter_mla_fp8_support
|
||||
|
||||
_install_fake_aiter_modules(monkeypatch, supports_fp8=True)
|
||||
|
||||
with patch("inspect.signature", side_effect=error_type("boom")):
|
||||
assert _check_aiter_mla_fp8_support() is False
|
||||
assert aiter_ops._AITER_MLA_SUPPORTS_FP8 is False
|
||||
|
||||
|
||||
def test_aiter_mla_fp8_support_result_is_cached(monkeypatch):
|
||||
"""The support check should reuse the cached result on later calls."""
|
||||
import inspect
|
||||
|
||||
from vllm._aiter_ops import _check_aiter_mla_fp8_support
|
||||
|
||||
_install_fake_aiter_modules(monkeypatch, supports_fp8=True)
|
||||
|
||||
with patch("inspect.signature", wraps=inspect.signature) as signature_mock:
|
||||
assert _check_aiter_mla_fp8_support() is True
|
||||
assert _check_aiter_mla_fp8_support() is True
|
||||
assert signature_mock.call_count == 1
|
||||
|
||||
|
||||
def test_missing_causal_arg_fails_closed(monkeypatch):
|
||||
"""Old aiter wheels have no causal=; do not report that as a valid draft path."""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLABackend
|
||||
|
||||
_install_fake_aiter_modules(monkeypatch, supports_fp8=True, supports_causal=False)
|
||||
with pytest.raises(RuntimeError, match="causal-only"):
|
||||
rocm_aiter_ops.mla_decode_supports_non_causal()
|
||||
with pytest.raises(RuntimeError, match="causal-only"):
|
||||
AiterMLABackend.supports_non_causal()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("has_kernels", [False, True])
|
||||
def test_supports_non_causal_requires_arch_kernels(monkeypatch, has_kernels):
|
||||
"""The Python argument alone does not guarantee an arch has the kernel."""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.v1.attention.backends.mla import rocm_aiter_mla
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLABackend
|
||||
|
||||
_install_fake_aiter_modules(monkeypatch, supports_fp8=True, supports_causal=True)
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla,
|
||||
"_aiter_mla_non_causal_asm_kernels",
|
||||
lambda: has_kernels,
|
||||
)
|
||||
assert rocm_aiter_ops.mla_decode_supports_non_causal() is True
|
||||
assert AiterMLABackend.supports_non_causal() is has_kernels
|
||||
|
||||
|
||||
@pytest.mark.parametrize("causal", [True, False])
|
||||
def test_decode_fwd_forwards_causal_to_aiter(monkeypatch, causal):
|
||||
"""The decode op must pass the block mask through; omitting it is causal-only."""
|
||||
seen: list = []
|
||||
|
||||
def mla_decode_fwd(*args, causal=None, **kwargs):
|
||||
seen.append(causal)
|
||||
|
||||
mla_mod: Any = types.ModuleType("aiter.mla")
|
||||
mla_mod.mla_decode_fwd = mla_decode_fwd
|
||||
aiter_mod: Any = types.ModuleType("aiter")
|
||||
aiter_mod.mla = mla_mod
|
||||
monkeypatch.setitem(sys.modules, "aiter", aiter_mod)
|
||||
monkeypatch.setitem(sys.modules, "aiter.mla", mla_mod)
|
||||
|
||||
import torch
|
||||
|
||||
import vllm._aiter_ops as aiter_ops
|
||||
|
||||
aiter_ops._rocm_aiter_mla_decode_fwd_impl(
|
||||
torch.zeros(2, 1, 8),
|
||||
torch.zeros(2, 1, 1, 8),
|
||||
torch.zeros(2, 1, 8),
|
||||
torch.zeros(3, dtype=torch.int32),
|
||||
1,
|
||||
causal=causal,
|
||||
)
|
||||
assert seen == [causal]
|
||||
|
||||
|
||||
def _aiter_mla_validate(
|
||||
monkeypatch,
|
||||
*,
|
||||
dtype,
|
||||
kv_cache_dtype,
|
||||
use_non_causal,
|
||||
):
|
||||
from vllm.platforms.interface import DeviceCapability
|
||||
from vllm.v1.attention.backend import AttentionType
|
||||
from vllm.v1.attention.backends.mla import rocm_aiter_mla
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLABackend
|
||||
|
||||
_install_fake_aiter_modules(monkeypatch, supports_fp8=True, supports_causal=True)
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla, "_aiter_mla_non_causal_asm_kernels", lambda: True
|
||||
)
|
||||
return AiterMLABackend.validate_configuration(
|
||||
head_size=576,
|
||||
dtype=dtype,
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
block_size=1,
|
||||
use_mla=True,
|
||||
has_sink=False,
|
||||
use_sparse=False,
|
||||
use_mm_prefix=False,
|
||||
use_per_head_quant_scales=False,
|
||||
device_capability=DeviceCapability(9, 5),
|
||||
attn_type=AttentionType.DECODER,
|
||||
use_non_causal=use_non_causal,
|
||||
)
|
||||
|
||||
|
||||
def test_non_causal_fp16_auto_cache_is_rejected(monkeypatch):
|
||||
"""Pinned AITER aborts fp16 Q; auto-select must fall through to Triton."""
|
||||
import torch
|
||||
|
||||
reasons = _aiter_mla_validate(
|
||||
monkeypatch,
|
||||
dtype=torch.float16,
|
||||
kv_cache_dtype="auto",
|
||||
use_non_causal=True,
|
||||
)
|
||||
assert any("non-causal fp16" in reason for reason in reasons)
|
||||
@@ -1,357 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Head padding and kernel selection for the ROCm AITER MLA backend.
|
||||
|
||||
The asm persistent decode requires a 16-aligned head count, so unaligned
|
||||
counts through 128 are tile-padded to the next multiple of 16 and sliced back
|
||||
off the output. Small divisor counts (1/2/4/8) preserve their existing
|
||||
repeat-interleave path and may keep the Gluon kernel on gfx950.
|
||||
"""
|
||||
|
||||
import math
|
||||
import types
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm._aiter_ops import is_aiter_found
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.mla import rocm_aiter_mla
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla import (
|
||||
AiterMLADecodeMetadata,
|
||||
AiterMLAHelper,
|
||||
AiterMLAImpl,
|
||||
AiterMLAMetadata,
|
||||
)
|
||||
|
||||
NUM_HEADS = 12
|
||||
KV_LORA_RANK = 512
|
||||
QK_ROPE_HEAD_DIM = 64
|
||||
QK_HEAD_DIM = KV_LORA_RANK + QK_ROPE_HEAD_DIM
|
||||
CONTEXT_LEN = 4096
|
||||
SCALE = 1.0 / math.sqrt(QK_HEAD_DIM)
|
||||
|
||||
# Small non-divisor counts use tile-and-slice; divisors of 16 keep
|
||||
# repeat_interleave. Both pad to exactly 16 and round-trip.
|
||||
NON_DIVISOR_HEADS = [3, 5, 6, 7, 9, 10, 11, 12, 13, 14, 15]
|
||||
DIVISOR_HEADS = [1, 2, 4, 8]
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _disable_native_h24(monkeypatch):
|
||||
"""Exercise the padding fallback unless a test explicitly enables H24."""
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla, "_aiter_mla_native_h24_supported", lambda: False
|
||||
)
|
||||
|
||||
|
||||
def _rocm_aiter_available() -> bool:
|
||||
return current_platform.is_rocm() and is_aiter_found() and torch.cuda.is_available()
|
||||
|
||||
|
||||
def _on_gfx950() -> bool:
|
||||
if not (current_platform.is_rocm() and is_aiter_found()):
|
||||
return False
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
|
||||
return on_gfx950()
|
||||
|
||||
|
||||
def _expected_tile_pad(q: torch.Tensor, num_heads: int, m: int = 16) -> torch.Tensor:
|
||||
# get_mla_padded_q tiles the heads and slices to m, i.e. head i of the
|
||||
# padded tensor is head (i % num_heads) of the input.
|
||||
idx = [i % num_heads for i in range(m)]
|
||||
return q[:, idx, :]
|
||||
|
||||
|
||||
def _make_h12_decode_metadata(device: torch.device) -> AiterMLAMetadata:
|
||||
page_indices = torch.arange(CONTEXT_LEN, dtype=torch.int32, device=device)
|
||||
decode = AiterMLADecodeMetadata(
|
||||
block_table=page_indices.view(1, -1),
|
||||
seq_lens=torch.tensor([CONTEXT_LEN], dtype=torch.int32, device=device),
|
||||
dcp_tot_seq_lens=None,
|
||||
paged_kv_indptr=torch.tensor(
|
||||
[0, CONTEXT_LEN], dtype=torch.int32, device=device
|
||||
),
|
||||
paged_kv_indices=page_indices,
|
||||
paged_kv_last_page_len=torch.ones(1, dtype=torch.int32, device=device),
|
||||
qo_indptr=torch.tensor([0, 1], dtype=torch.int32, device=device),
|
||||
attn_out_dtype=torch.bfloat16,
|
||||
max_qo_len=1,
|
||||
)
|
||||
return AiterMLAMetadata(
|
||||
num_reqs=1,
|
||||
max_query_len=1,
|
||||
max_seq_len=CONTEXT_LEN,
|
||||
num_actual_tokens=1,
|
||||
query_start_loc=decode.qo_indptr,
|
||||
slot_mapping=torch.tensor([CONTEXT_LEN - 1], dtype=torch.int64, device=device),
|
||||
num_decodes=1,
|
||||
num_decode_tokens=1,
|
||||
num_prefills=0,
|
||||
decode=decode,
|
||||
)
|
||||
|
||||
|
||||
def test_h12_query_is_tile_padded_to_h16():
|
||||
q = torch.arange(2 * 12 * 4, dtype=torch.bfloat16).view(2, 12, 4)
|
||||
|
||||
padded_q = AiterMLAHelper.get_mla_padded_q(12, q)
|
||||
|
||||
assert padded_q.shape == (2, 16, 4)
|
||||
assert padded_q.is_contiguous()
|
||||
# The real heads are untouched...
|
||||
torch.testing.assert_close(padded_q[:, :12], q)
|
||||
# ...and the 4 padding heads are the tiled wrap-around (heads 0..3), not
|
||||
# zeros: MLA attention is per-head independent so duplicate query heads are
|
||||
# harmless and get sliced back off the output.
|
||||
torch.testing.assert_close(padded_q[:, 12:], q[:, :4])
|
||||
|
||||
|
||||
def test_h6_tp16_query_is_padded_to_h16():
|
||||
# TP16 puts 6 heads/rank. The old append-only padding produced cat(6, 6) =
|
||||
# 12 heads and broke the asm kernel; tile-and-slice must reach exactly 16.
|
||||
q = torch.arange(2 * 6 * 4, dtype=torch.bfloat16).view(2, 6, 4)
|
||||
|
||||
padded_q = AiterMLAHelper.get_mla_padded_q(6, q)
|
||||
|
||||
assert padded_q.shape == (2, 16, 4)
|
||||
assert padded_q.is_contiguous()
|
||||
torch.testing.assert_close(padded_q, _expected_tile_pad(q, 6))
|
||||
|
||||
|
||||
def test_h12_output_discards_padding_heads():
|
||||
o = torch.arange(2 * 16 * 4, dtype=torch.bfloat16).view(2, 16, 4)
|
||||
|
||||
unpadded_o = AiterMLAHelper.get_mla_unpadded_o(12, o)
|
||||
|
||||
assert unpadded_o.shape == (2, 12, 4)
|
||||
torch.testing.assert_close(unpadded_o, o[:, :12])
|
||||
|
||||
|
||||
def test_h24_query_is_tile_padded_to_h32():
|
||||
q = torch.arange(2 * 24 * 4, dtype=torch.float32).view(2, 24, 4)
|
||||
|
||||
padded_q = AiterMLAHelper.get_mla_padded_q(24, q)
|
||||
|
||||
assert padded_q.shape == (2, 32, 4)
|
||||
assert padded_q.is_contiguous()
|
||||
torch.testing.assert_close(padded_q[:, :24], q)
|
||||
torch.testing.assert_close(padded_q[:, 24:], q[:, :8])
|
||||
|
||||
|
||||
def test_h24_output_discards_h32_padding_heads():
|
||||
o = torch.arange(2 * 32 * 4, dtype=torch.float32).view(2, 32, 4)
|
||||
|
||||
unpadded_o = AiterMLAHelper.get_mla_unpadded_o(24, o)
|
||||
|
||||
assert unpadded_o.shape == (2, 24, 4)
|
||||
torch.testing.assert_close(unpadded_o, o[:, :24])
|
||||
|
||||
|
||||
def test_h24_reducer_without_metadata_still_pads_to_h32(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla, "_aiter_mla_native_h24_reducer_supported", lambda: True
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla, "_aiter_mla_native_h24_metadata_supported", lambda: False
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla,
|
||||
"_aiter_mla_native_h24_supported",
|
||||
lambda: (
|
||||
rocm_aiter_mla._aiter_mla_native_h24_reducer_supported()
|
||||
and rocm_aiter_mla._aiter_mla_native_h24_metadata_supported()
|
||||
),
|
||||
)
|
||||
q = torch.arange(2 * 24 * 4, dtype=torch.float32).view(2, 24, 4)
|
||||
|
||||
assert AiterMLAHelper.get_actual_mla_num_heads(24) == 32
|
||||
padded_q = AiterMLAHelper.get_mla_padded_q(24, q)
|
||||
assert padded_q.shape == (2, 32, 4)
|
||||
torch.testing.assert_close(padded_q[:, :24], q)
|
||||
torch.testing.assert_close(padded_q[:, 24:], q[:, :8])
|
||||
|
||||
|
||||
def test_native_h24_requires_reducer_and_metadata(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla, "_aiter_mla_native_h24_reducer_supported", lambda: True
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla, "_aiter_mla_native_h24_metadata_supported", lambda: True
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla,
|
||||
"_aiter_mla_native_h24_supported",
|
||||
lambda: (
|
||||
rocm_aiter_mla._aiter_mla_native_h24_reducer_supported()
|
||||
and rocm_aiter_mla._aiter_mla_native_h24_metadata_supported()
|
||||
),
|
||||
)
|
||||
q = torch.arange(2 * 24 * 4, dtype=torch.float32).view(2, 24, 4)
|
||||
|
||||
assert AiterMLAHelper.get_actual_mla_num_heads(24) == 24
|
||||
assert AiterMLAHelper.get_mla_padded_q(24, q) is q
|
||||
assert AiterMLAHelper.get_mla_unpadded_o(24, q) is q
|
||||
|
||||
|
||||
def test_existing_divisor_head_mapping_is_unchanged():
|
||||
q = torch.arange(2 * 8 * 4, dtype=torch.bfloat16).view(2, 8, 4)
|
||||
|
||||
padded_q = AiterMLAHelper.get_mla_padded_q(8, q)
|
||||
unpadded_o = AiterMLAHelper.get_mla_unpadded_o(8, padded_q)
|
||||
|
||||
# Divisor counts still use repeat_interleave / strided unpad, unchanged.
|
||||
torch.testing.assert_close(padded_q, q.repeat_interleave(2, dim=1))
|
||||
torch.testing.assert_close(unpadded_o, q)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_heads", [17, 24, 31])
|
||||
def test_unaligned_head_counts_round_trip_through_h32(num_heads: int):
|
||||
q = torch.arange(2 * num_heads * 4, dtype=torch.float32).view(2, num_heads, 4)
|
||||
|
||||
padded_q = AiterMLAHelper.get_mla_padded_q(num_heads, q)
|
||||
unpadded_o = AiterMLAHelper.get_mla_unpadded_o(num_heads, padded_q)
|
||||
|
||||
assert padded_q.shape == (2, 32, 4)
|
||||
assert padded_q.is_contiguous()
|
||||
torch.testing.assert_close(unpadded_o, q)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_heads", NON_DIVISOR_HEADS + DIVISOR_HEADS)
|
||||
def test_all_small_head_counts_pad_to_16_and_round_trip(num_heads: int):
|
||||
q = torch.arange(2 * num_heads * 4, dtype=torch.float32).view(2, num_heads, 4)
|
||||
|
||||
padded_q = AiterMLAHelper.get_mla_padded_q(num_heads, q)
|
||||
unpadded_o = AiterMLAHelper.get_mla_unpadded_o(num_heads, padded_q)
|
||||
|
||||
assert padded_q.shape == (2, 16, 4)
|
||||
assert padded_q.is_contiguous()
|
||||
# The real heads survive the pad -> unpad round trip exactly.
|
||||
torch.testing.assert_close(unpadded_o, q)
|
||||
|
||||
|
||||
def test_aligned_h32_is_zero_copy():
|
||||
q = torch.arange(2 * 32 * 4, dtype=torch.float32).view(2, 32, 4)
|
||||
assert AiterMLAHelper.get_mla_padded_q(32, q) is q
|
||||
assert AiterMLAHelper.get_mla_unpadded_o(32, q) is q
|
||||
|
||||
|
||||
def test_is_valid_num_heads():
|
||||
for n in range(1, 129):
|
||||
assert AiterMLAHelper.is_valid_num_heads(n)
|
||||
assert AiterMLAHelper.is_valid_num_heads(24)
|
||||
assert AiterMLAHelper.is_valid_num_heads(127)
|
||||
# Aligned counts remain valid above the range where padding is supported.
|
||||
assert AiterMLAHelper.is_valid_num_heads(144)
|
||||
assert not AiterMLAHelper.is_valid_num_heads(0)
|
||||
assert not AiterMLAHelper.is_valid_num_heads(129)
|
||||
|
||||
|
||||
def test_nondivisor_and_multitoken_never_use_gluon():
|
||||
# Non-divisor decode always takes the asm path (12 heads/rank at TP8).
|
||||
assert not AiterMLAHelper.use_gluon_decode(12, 1, "auto")
|
||||
assert not AiterMLAHelper.use_gluon_decode(6, 1, "auto")
|
||||
# >=16 heads never use Gluon, including unaligned counts padded for asm.
|
||||
assert not AiterMLAHelper.use_gluon_decode(16, 1, "auto")
|
||||
assert not AiterMLAHelper.use_gluon_decode(24, 1, "auto")
|
||||
# Multi-token (verify / qlen>1) is never the single-token Gluon decode.
|
||||
assert not AiterMLAHelper.use_gluon_decode(8, 4, "auto")
|
||||
assert not AiterMLAHelper.use_gluon_decode(12, 4, "auto")
|
||||
|
||||
|
||||
def test_divisor_gluon_selection_follows_arch():
|
||||
# Divisor head counts keep Gluon only where the kernel exists (gfx950).
|
||||
# On gfx942 / non-ROCm they must route to the asm persistent decode.
|
||||
on_gfx950 = _on_gfx950()
|
||||
assert AiterMLAHelper.use_gluon_decode(8, 1, "auto") is on_gfx950
|
||||
assert AiterMLAHelper.use_gluon_decode(4, 1, "auto") is on_gfx950
|
||||
assert AiterMLAHelper.use_gluon_decode(1, 1, "auto") is on_gfx950
|
||||
|
||||
|
||||
def test_asm_padding_env_default_is_auto(monkeypatch):
|
||||
monkeypatch.delenv("VLLM_ROCM_AITER_MLA_ASM_PADDING", raising=False)
|
||||
import vllm.envs as envs
|
||||
|
||||
assert envs.VLLM_ROCM_AITER_MLA_ASM_PADDING == "auto"
|
||||
|
||||
|
||||
def test_asm_padding_env_force_asm_disables_gluon(monkeypatch):
|
||||
monkeypatch.setenv("VLLM_ROCM_AITER_MLA_ASM_PADDING", "asm")
|
||||
# Forcing the asm path: no small-head count uses Gluon on any arch.
|
||||
for num_heads in (1, 2, 4, 8, 12):
|
||||
assert not AiterMLAHelper.use_gluon_decode(num_heads, 1, "auto")
|
||||
|
||||
|
||||
def test_asm_padding_env_force_gluon_follows_arch(monkeypatch):
|
||||
monkeypatch.setenv("VLLM_ROCM_AITER_MLA_ASM_PADDING", "gluon")
|
||||
on_gfx950 = _on_gfx950()
|
||||
# Forcing Gluon: any 1..15 single-token decode uses it where a build exists
|
||||
# (gfx950), including non-divisor counts like 12; gfx942/non-ROCm still
|
||||
# falls back to the asm path.
|
||||
for num_heads in (1, 2, 4, 8, 12):
|
||||
assert AiterMLAHelper.use_gluon_decode(num_heads, 1, "auto") is on_gfx950
|
||||
|
||||
|
||||
def test_asm_padding_env_auto_matches_arch_gate(monkeypatch):
|
||||
monkeypatch.setenv("VLLM_ROCM_AITER_MLA_ASM_PADDING", "auto")
|
||||
on_gfx950 = _on_gfx950()
|
||||
# auto: divisor counts keep Gluon on gfx950, non-divisor counts take asm.
|
||||
assert AiterMLAHelper.use_gluon_decode(8, 1, "auto") is on_gfx950
|
||||
assert not AiterMLAHelper.use_gluon_decode(12, 1, "auto")
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not _rocm_aiter_available(),
|
||||
reason="12-head AITER MLA asm persistent decode needs ROCm + AITER",
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_h12_aiter_mla_decode_matches_reference():
|
||||
"""The 12-head AITER decode output must match an attention reference.
|
||||
|
||||
12 is a non-divisor of 16 so use_gluon_decode is False on every arch; the
|
||||
query is tile-padded to 16 heads and served by the asm persistent decode
|
||||
(the CDNA3 path this PR enables, and unchanged on CDNA4).
|
||||
"""
|
||||
device = torch.device("cuda:0")
|
||||
torch.manual_seed(42)
|
||||
|
||||
q = torch.randn(1, NUM_HEADS, QK_HEAD_DIM, dtype=torch.bfloat16, device=device)
|
||||
kv_cache = torch.randn(
|
||||
CONTEXT_LEN, 1, QK_HEAD_DIM, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
|
||||
impl = object.__new__(AiterMLAImpl)
|
||||
impl.num_heads = NUM_HEADS
|
||||
impl.dcp_world_size = 1
|
||||
impl.kv_lora_rank = KV_LORA_RANK
|
||||
impl.qk_rope_head_dim = QK_ROPE_HEAD_DIM
|
||||
impl.scale = SCALE
|
||||
impl.kv_cache_dtype = "auto"
|
||||
|
||||
one = torch.ones(1, dtype=torch.float32, device=device)
|
||||
layer = types.SimpleNamespace(_q_scale=one, _k_scale=one)
|
||||
metadata = _make_h12_decode_metadata(device)
|
||||
|
||||
out, _ = impl.forward_mqa(q, kv_cache, metadata, layer)
|
||||
|
||||
assert out.shape[1] == NUM_HEADS # padding heads sliced back off
|
||||
|
||||
key = kv_cache[:, 0].float().unsqueeze(0)
|
||||
value = key[..., :KV_LORA_RANK]
|
||||
out_ref = F.scaled_dot_product_attention(
|
||||
q[0].float().unsqueeze(1),
|
||||
key,
|
||||
value,
|
||||
scale=SCALE,
|
||||
enable_gqa=True,
|
||||
).squeeze(1)
|
||||
|
||||
assert torch.isfinite(out).all()
|
||||
torch.testing.assert_close(
|
||||
out.float(),
|
||||
out_ref.unsqueeze(0),
|
||||
atol=1e-2,
|
||||
rtol=1e-2,
|
||||
)
|
||||
@@ -1,523 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Correctness tests for ROCm AITER sparse MLA attention sinks."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_rocm(), reason="ROCm-specific tests"
|
||||
)
|
||||
|
||||
Q_HEAD_DIM = 576
|
||||
V_HEAD_DIM = 512
|
||||
SM_SCALE = Q_HEAD_DIM**-0.5
|
||||
|
||||
|
||||
def _gamma_fp32(operations: int) -> float:
|
||||
u = torch.finfo(torch.float32).eps / 2
|
||||
return operations * u / (1 - operations * u)
|
||||
|
||||
|
||||
def _sink_reference(q, keys, sinks, scale):
|
||||
# Reference the stored operands, so input quantization is not kernel error.
|
||||
q, keys, sinks = q.double(), keys.double(), sinks.double()
|
||||
scores = q @ keys.T * scale
|
||||
logits = torch.cat((scores, sinks[:, None]), dim=-1)
|
||||
probabilities = logits.softmax(dim=-1)[:, :-1]
|
||||
values = keys[:, :V_HEAD_DIM]
|
||||
expected = probabilities @ values
|
||||
value_scale = probabilities @ values.abs()
|
||||
# FP32 dot accumulation and conversion/multiplication of the scale.
|
||||
score_error = _gamma_fp32(q.shape[-1] + 2) * (q.abs() @ keys.abs().T) * abs(scale)
|
||||
score_error = score_error.amax(dim=-1) if keys.shape[0] else torch.zeros_like(sinks)
|
||||
return expected, logits.logsumexp(dim=-1), value_scale, score_error
|
||||
|
||||
|
||||
def _assert_sink_output_close(
|
||||
actual, expected, value_scale, score_error, probability_dtype, native
|
||||
):
|
||||
assert actual.shape == expected.shape
|
||||
# Budget the identifiable rounding stages against the absolute value
|
||||
# contributions: output-relative ULPs become unbounded when values cancel.
|
||||
# Rounding P before P@V costs u(P) * sum(P*abs(V)).
|
||||
# AITER rounds output before and after sink scaling; Triton rounds once.
|
||||
u_probability = torch.finfo(probability_dtype).eps / 2
|
||||
u_output = torch.finfo(actual.dtype).eps / 2
|
||||
output_rounds = 2 if native else 1
|
||||
amplification = (1 + u_output) ** output_rounds
|
||||
relative_error = (1 + u_probability) * torch.exp(2 * score_error) - 1
|
||||
allowance = (
|
||||
relative_error.unsqueeze(-1) * value_scale * amplification
|
||||
+ (amplification - 1) * expected.abs()
|
||||
+ output_rounds * torch.finfo(actual.dtype).tiny * u_output
|
||||
)
|
||||
actual = actual.to(device=expected.device, dtype=torch.float64)
|
||||
assert torch.isfinite(actual).all()
|
||||
assert torch.count_nonzero(actual[value_scale == 0]) == 0
|
||||
error = (actual - expected).abs()
|
||||
max_ratio = (error / allowance.clamp_min(1e-300)).max().item()
|
||||
assert max_ratio <= 1, f"exceeded rounding budget by {max_ratio:.3f}x"
|
||||
|
||||
|
||||
def _require_aiter() -> None:
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported
|
||||
from vllm.platforms.rocm import get_cdna_version
|
||||
|
||||
if get_cdna_version() not in (3, 4):
|
||||
pytest.skip("AITER MLA requires CDNA 3 or 4")
|
||||
|
||||
if not is_aiter_found_and_supported():
|
||||
pytest.skip("aiter is required on supported ROCm hardware for this test")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("real_heads", "cache_kind"),
|
||||
[
|
||||
(1, "bf16"),
|
||||
(4, "bf16"),
|
||||
(8, "bf16"),
|
||||
(8, "fp8"),
|
||||
(32, "fp8"),
|
||||
(64, "fp8"),
|
||||
(12, "bf16"),
|
||||
(16, "bf16"),
|
||||
(24, "bf16"),
|
||||
(32, "bf16"),
|
||||
(40, "bf16"),
|
||||
(48, "bf16"),
|
||||
(64, "bf16"),
|
||||
(80, "bf16"),
|
||||
],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_sparse_mla_sink_matches_ragged_reference(
|
||||
real_heads: int, cache_kind: str
|
||||
) -> None:
|
||||
"""Exercise ragged sink decode, head padding, and FP8 scale forwarding."""
|
||||
_require_aiter()
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAHelper
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import (
|
||||
ROCMAiterMLASparseImpl,
|
||||
)
|
||||
|
||||
set_random_seed(real_heads * 17 + (cache_kind == "fp8"))
|
||||
device = torch.device("cuda")
|
||||
seq_lens = [0, 1, 5, 23, 17]
|
||||
batch_size = len(seq_lens)
|
||||
|
||||
q_source = torch.randn(batch_size, real_heads, Q_HEAD_DIM, device=device) * 0.2
|
||||
|
||||
pool_size = sum(seq_lens) + 52
|
||||
kv_source = torch.randn(pool_size, 1, Q_HEAD_DIM, device=device) * 0.2
|
||||
if cache_kind == "fp8":
|
||||
fp8_dtype = current_platform.fp8_dtype()
|
||||
q_scale = torch.tensor(0.5, dtype=torch.float32, device=device)
|
||||
kv_scale = torch.tensor(0.25, dtype=torch.float32, device=device)
|
||||
q_real = (q_source / q_scale).to(fp8_dtype)
|
||||
kv = (kv_source / kv_scale).to(fp8_dtype)
|
||||
q_ref = q_real.float() * q_scale
|
||||
kv_ref = kv.float() * kv_scale
|
||||
else:
|
||||
q_scale = kv_scale = None
|
||||
q_real = q_source.to(torch.bfloat16)
|
||||
kv = kv_source.to(torch.bfloat16)
|
||||
q_ref = q_real.float()
|
||||
kv_ref = kv.float()
|
||||
q = AiterMLAHelper.get_mla_padded_q(real_heads, q_real)
|
||||
|
||||
indices = torch.randperm(pool_size, device=device)[: sum(seq_lens)].to(torch.int32)
|
||||
kv_indptr = torch.tensor(
|
||||
[0] + [sum(seq_lens[:i]) for i in range(1, len(seq_lens) + 1)],
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
|
||||
# Non-None garbage proves the sink path cannot accidentally select the
|
||||
# gfx942 persistent kernel, which has no return-LSE code object.
|
||||
metadata = SimpleNamespace(
|
||||
attn_out_dtype=torch.bfloat16,
|
||||
qo_indptr=torch.arange(batch_size + 1, dtype=torch.int32, device=device),
|
||||
paged_kv_indptr=kv_indptr,
|
||||
paged_kv_indices=indices,
|
||||
paged_kv_last_page_len=torch.ones(batch_size, dtype=torch.int32, device=device),
|
||||
work_meta_data=torch.tensor([123], dtype=torch.int32),
|
||||
work_indptr=None,
|
||||
work_info_set=None,
|
||||
reduce_indptr=None,
|
||||
reduce_final_map=None,
|
||||
reduce_partial_map=None,
|
||||
num_prefills=0,
|
||||
num_decodes=batch_size,
|
||||
num_decode_tokens=batch_size,
|
||||
max_query_len=1,
|
||||
max_seq_len=max(seq_lens),
|
||||
topk_tokens=max(seq_lens),
|
||||
)
|
||||
sinks = torch.linspace(-2.0, 6.0, real_heads, device=device)
|
||||
|
||||
impl = object.__new__(ROCMAiterMLASparseImpl)
|
||||
impl.num_heads = real_heads
|
||||
impl.kv_lora_rank = V_HEAD_DIM
|
||||
impl.kv_cache_dtype = cache_kind
|
||||
impl.scale = SM_SCALE
|
||||
impl.sinks = sinks
|
||||
|
||||
output, lse = impl._forward_mla(
|
||||
SimpleNamespace(_q_scale=q_scale, _k_scale=kv_scale),
|
||||
q,
|
||||
kv,
|
||||
metadata,
|
||||
)
|
||||
kv_flat = kv_ref[:, 0]
|
||||
references = []
|
||||
start = 0
|
||||
for batch_idx, seq_len in enumerate(seq_lens):
|
||||
rows = kv_flat[indices[start : start + seq_len].long()]
|
||||
references.append(_sink_reference(q_ref[batch_idx], rows, sinks, SM_SCALE))
|
||||
start += seq_len
|
||||
expected, expected_lse, value_scale, score_error = (
|
||||
torch.stack(values) for values in zip(*references)
|
||||
)
|
||||
|
||||
assert output.dtype == torch.bfloat16
|
||||
if lse is not None:
|
||||
assert lse.dtype == torch.float32
|
||||
assert lse.shape == expected_lse.shape
|
||||
lse_error = (lse.double() - expected_lse).abs()
|
||||
rounded_lse = expected_lse.float()
|
||||
positive_inf = torch.full_like(rounded_lse, float("inf"))
|
||||
ulp = torch.maximum(
|
||||
torch.nextafter(rounded_lse, positive_inf) - rounded_lse,
|
||||
rounded_lse - torch.nextafter(rounded_lse, -positive_inf),
|
||||
).double()
|
||||
lse_allowance = score_error + 2 * ulp
|
||||
assert torch.all(lse_error <= lse_allowance)
|
||||
torch.testing.assert_close(lse[0].double(), expected_lse[0], atol=0, rtol=0)
|
||||
else:
|
||||
from vllm.platforms.rocm import on_gfx942
|
||||
|
||||
assert cache_kind == "bf16" and real_heads in (40, 48, 64) and on_gfx942()
|
||||
_assert_sink_output_close(
|
||||
output,
|
||||
expected,
|
||||
value_scale,
|
||||
score_error,
|
||||
q_real.dtype,
|
||||
native=lse is not None,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("q_dtype", "kv_dtype"),
|
||||
[
|
||||
(torch.float16, torch.bfloat16),
|
||||
(torch.bfloat16, torch.float16),
|
||||
],
|
||||
)
|
||||
def test_sparse_mla_sink_rejects_unsupported_aiter_dtypes(
|
||||
q_dtype: torch.dtype, kv_dtype: torch.dtype
|
||||
) -> None:
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import (
|
||||
ROCMAiterMLASparseImpl,
|
||||
)
|
||||
|
||||
impl = object.__new__(ROCMAiterMLASparseImpl)
|
||||
impl.num_heads = 16
|
||||
impl.kv_lora_rank = V_HEAD_DIM
|
||||
impl.kv_cache_dtype = "auto"
|
||||
impl.scale = SM_SCALE
|
||||
impl.sinks = torch.zeros(16, dtype=torch.float32)
|
||||
metadata = SimpleNamespace(
|
||||
attn_out_dtype=torch.bfloat16,
|
||||
num_prefills=0,
|
||||
num_decodes=1,
|
||||
num_decode_tokens=1,
|
||||
max_query_len=1,
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="both use BF16 or both use FP8"):
|
||||
impl._forward_mla(
|
||||
SimpleNamespace(_q_scale=None, _k_scale=None),
|
||||
torch.empty(1, 16, Q_HEAD_DIM, dtype=q_dtype),
|
||||
torch.empty(1, 1, Q_HEAD_DIM, dtype=kv_dtype),
|
||||
metadata,
|
||||
)
|
||||
|
||||
|
||||
def _make_noncontiguous_sink() -> torch.Tensor:
|
||||
return torch.empty(8, dtype=torch.float32)[::2]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("sinks", "match"),
|
||||
[
|
||||
(torch.empty(4, dtype=torch.bfloat16), "must be float32"),
|
||||
(torch.empty(2, 2, dtype=torch.float32), "must have shape"),
|
||||
(_make_noncontiguous_sink(), "must be contiguous"),
|
||||
],
|
||||
)
|
||||
def test_sparse_mla_sink_validation(sinks: torch.Tensor, match: str) -> None:
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import (
|
||||
ROCMAiterMLASparseImpl,
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match=match):
|
||||
ROCMAiterMLASparseImpl(
|
||||
num_heads=4,
|
||||
head_size=Q_HEAD_DIM,
|
||||
scale=SM_SCALE,
|
||||
num_kv_heads=1,
|
||||
alibi_slopes=None,
|
||||
sliding_window=None,
|
||||
kv_cache_dtype="bfloat16",
|
||||
logits_soft_cap=None,
|
||||
attn_type="decoder",
|
||||
kv_sharing_target_layer_name=None,
|
||||
sinks=sinks,
|
||||
kv_lora_rank=V_HEAD_DIM,
|
||||
)
|
||||
|
||||
|
||||
def test_sparse_mla_backend_reports_sink_support_for_current_hardware() -> None:
|
||||
from vllm.platforms.rocm import get_cdna_version
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import (
|
||||
ROCMAiterMLASparseBackend,
|
||||
)
|
||||
|
||||
assert ROCMAiterMLASparseBackend.supports_sink() == (get_cdna_version() in (3, 4))
|
||||
|
||||
|
||||
def test_sparse_mla_backend_rejects_dcp() -> None:
|
||||
from vllm.platforms.rocm import RocmPlatform
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.attention.selector import AttentionSelectorConfig
|
||||
|
||||
selector_config = AttentionSelectorConfig(
|
||||
head_size=Q_HEAD_DIM,
|
||||
dtype=torch.bfloat16,
|
||||
kv_cache_dtype="bfloat16",
|
||||
block_size=16,
|
||||
use_mla=True,
|
||||
has_sink=True,
|
||||
use_sparse=True,
|
||||
use_mm_prefix=False,
|
||||
use_per_head_quant_scales=False,
|
||||
attn_type="decoder",
|
||||
use_dcp=True,
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="DCP not supported"):
|
||||
RocmPlatform.get_attn_backend_cls(
|
||||
selected_backend=AttentionBackendEnum.ROCM_AITER_MLA_SPARSE,
|
||||
attn_selector_config=selector_config,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_heads,block_size", [(8, 16), (20, 32)])
|
||||
@pytest.mark.parametrize("interleaved_pages", [False, True])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
||||
@torch.inference_mode()
|
||||
def test_sparse_mla_sink_matches_dense_attention_with_empty_rows_and_paged_cache(
|
||||
num_heads: int,
|
||||
block_size: int,
|
||||
interleaved_pages: bool,
|
||||
dtype: torch.dtype,
|
||||
) -> None:
|
||||
"""Preserve head-specific sinks and latent values across ragged cache pages."""
|
||||
_require_aiter()
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAHelper
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import (
|
||||
ROCMAiterMLASparseImpl,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.sparse_utils import flat_kv_row_view
|
||||
|
||||
head_dim, value_dim = 576, 512
|
||||
generator = torch.Generator().manual_seed(421)
|
||||
# The empty row exercises sink-only attention; the last row crosses both
|
||||
# page boundaries and the kernel's 16-key reduction tile.
|
||||
selected_rows = [
|
||||
[],
|
||||
[(2, 7)],
|
||||
[(3, 1), (0, block_size - 1), (1, 0), (2, 4)],
|
||||
[(i % 4, (i * 7) % block_size) for i in range(23)],
|
||||
]
|
||||
q_cpu = torch.randn(
|
||||
len(selected_rows), num_heads, head_dim, generator=generator
|
||||
).to(dtype)
|
||||
cache_cpu = torch.randn(4, block_size, head_dim, generator=generator).to(dtype)
|
||||
sinks_cpu = torch.linspace(-4.0, 6.0, num_heads)
|
||||
|
||||
# Layers can share a backing allocation, leaving unused rows between pages.
|
||||
num_layers = 3 if interleaved_pages else 1
|
||||
backing = torch.full(
|
||||
(4, num_layers, block_size, head_dim),
|
||||
float("nan"),
|
||||
dtype=dtype,
|
||||
device="cuda",
|
||||
)
|
||||
cache = backing[:, num_layers - 1]
|
||||
cache.copy_(cache_cpu)
|
||||
indices = [
|
||||
page * num_layers * block_size + offset
|
||||
for rows in selected_rows
|
||||
for page, offset in rows
|
||||
]
|
||||
lengths = torch.tensor([0, *(len(rows) for rows in selected_rows)])
|
||||
longest_row = max(len(rows) for rows in selected_rows)
|
||||
metadata = SimpleNamespace(
|
||||
block_size=block_size,
|
||||
num_prefills=0,
|
||||
num_decodes=len(selected_rows),
|
||||
num_decode_tokens=len(selected_rows),
|
||||
max_query_len=1,
|
||||
max_seq_len=longest_row,
|
||||
topk_tokens=longest_row,
|
||||
qo_indptr=torch.arange(
|
||||
len(selected_rows) + 1, dtype=torch.int32, device="cuda"
|
||||
),
|
||||
paged_kv_last_page_len=torch.ones(
|
||||
len(selected_rows), dtype=torch.int32, device="cuda"
|
||||
),
|
||||
work_meta_data=None,
|
||||
attn_out_dtype=dtype,
|
||||
paged_kv_indices=torch.tensor(indices, dtype=torch.int32, device="cuda"),
|
||||
paged_kv_indptr=lengths.cumsum(0).to(device="cuda", dtype=torch.int32),
|
||||
)
|
||||
impl = ROCMAiterMLASparseImpl.__new__(ROCMAiterMLASparseImpl)
|
||||
impl.num_heads = num_heads
|
||||
impl.head_size = head_dim
|
||||
impl.kv_lora_rank = value_dim
|
||||
impl.scale = head_dim**-0.5
|
||||
impl.kv_cache_dtype = "auto"
|
||||
impl.sinks = sinks_cpu.cuda()
|
||||
padded_q = AiterMLAHelper.get_mla_padded_q(num_heads, q_cpu.cuda())
|
||||
|
||||
kv_rows, _ = flat_kv_row_view(cache, block_size)
|
||||
actual, _ = impl._forward_mla(
|
||||
SimpleNamespace(_q_scale=None, _k_scale=None),
|
||||
padded_q,
|
||||
kv_rows.unsqueeze(1),
|
||||
metadata,
|
||||
)
|
||||
|
||||
references = []
|
||||
for query_idx, rows in enumerate(selected_rows):
|
||||
keys = (
|
||||
torch.stack([cache_cpu[page, offset] for page, offset in rows])
|
||||
if rows
|
||||
else cache_cpu.new_empty((0, head_dim))
|
||||
)
|
||||
references.append(
|
||||
_sink_reference(q_cpu[query_idx], keys, sinks_cpu, impl.scale)
|
||||
)
|
||||
expected, _, value_scale, score_error = (
|
||||
torch.stack(values) for values in zip(*references)
|
||||
)
|
||||
assert actual.shape == (len(selected_rows), num_heads, value_dim)
|
||||
assert actual.dtype == dtype
|
||||
_assert_sink_output_close(
|
||||
actual,
|
||||
expected,
|
||||
value_scale,
|
||||
score_error,
|
||||
dtype,
|
||||
native=dtype == torch.bfloat16,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("layout", ["LBNHC", "LBHNC", "BLNHC"])
|
||||
def test_sparse_mla_backend_resolves_only_contiguous_layer_layouts(monkeypatch, layout):
|
||||
from vllm.config import CacheConfig
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import (
|
||||
ROCMAiterMLASparseBackend,
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import resolve_kv_cache_layout
|
||||
|
||||
monkeypatch.setenv("VLLM_KV_CACHE_LAYOUT", layout)
|
||||
config = SimpleNamespace(cache_config=CacheConfig())
|
||||
supported = [
|
||||
[x.name for x in ROCMAiterMLASparseBackend.supported_kv_cache_layouts()]
|
||||
]
|
||||
if layout == "BLNHC":
|
||||
with pytest.raises(ValueError, match="does not satisfy"):
|
||||
resolve_kv_cache_layout(config, supported)
|
||||
else:
|
||||
assert resolve_kv_cache_layout(config, supported).name == layout
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
||||
@torch.inference_mode()
|
||||
def test_sparse_mla_sink_forward_mqa_preserves_split_query(dtype):
|
||||
"""The public sparse forward joins latent/RoPE queries in the model dtype."""
|
||||
_require_aiter()
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import (
|
||||
ROCMAiterMLASparseImpl,
|
||||
)
|
||||
|
||||
set_random_seed(412)
|
||||
num_tokens, num_heads, block_size = 2, 8, 16
|
||||
q = torch.randn(num_tokens, num_heads, Q_HEAD_DIM, device="cuda").to(dtype)
|
||||
kv = torch.randn(2, 2, block_size, Q_HEAD_DIM, device="cuda").to(dtype)[:, 1]
|
||||
selected = torch.tensor([[1, 7, 18], [0, 3, 20]], dtype=torch.int32, device="cuda")
|
||||
sinks = torch.linspace(-3.0, 5.0, num_heads, device="cuda")
|
||||
impl = object.__new__(ROCMAiterMLASparseImpl)
|
||||
impl.num_heads = num_heads
|
||||
impl.kv_lora_rank = V_HEAD_DIM
|
||||
impl.kv_cache_dtype = "auto"
|
||||
impl.scale = SM_SCALE
|
||||
impl.sinks = sinks
|
||||
impl.topk_indices_buffer = torch.full(
|
||||
(num_tokens, 128), -1, dtype=torch.int32, device="cuda"
|
||||
)
|
||||
impl.topk_indices_buffer[:, : selected.shape[1]] = selected
|
||||
impl.owns_indexer = True
|
||||
impl.index_epoch = 0
|
||||
impl.q_concat_buffer = torch.empty_like(q)
|
||||
metadata = SimpleNamespace(
|
||||
attn_out_dtype=dtype,
|
||||
num_actual_tokens=num_tokens,
|
||||
num_prefills=0,
|
||||
num_decodes=num_tokens,
|
||||
num_decode_tokens=num_tokens,
|
||||
max_query_len=1,
|
||||
max_seq_len=kv.shape[0],
|
||||
block_size=block_size,
|
||||
topk_tokens=impl.topk_indices_buffer.shape[1],
|
||||
req_id_per_token=torch.zeros(num_tokens, dtype=torch.int32, device="cuda"),
|
||||
block_table=torch.tensor([[0, 1]], dtype=torch.int32, device="cuda"),
|
||||
qo_indptr=torch.arange(num_tokens + 1, dtype=torch.int32, device="cuda"),
|
||||
paged_kv_indptr=torch.tensor([0, 3, 6], dtype=torch.int32, device="cuda"),
|
||||
paged_kv_indices=torch.empty(
|
||||
selected.numel(), dtype=torch.int32, device="cuda"
|
||||
),
|
||||
paged_kv_last_page_len=torch.ones(num_tokens, dtype=torch.int32, device="cuda"),
|
||||
work_meta_data=None,
|
||||
)
|
||||
actual, _ = impl.forward_mqa(
|
||||
(q[..., :V_HEAD_DIM], q[..., V_HEAD_DIM:]),
|
||||
kv,
|
||||
metadata,
|
||||
SimpleNamespace(_q_scale=None, _k_scale=None),
|
||||
)
|
||||
references = [
|
||||
_sink_reference(q[i], kv.flatten(0, 1)[selected[i].long()], sinks, SM_SCALE)
|
||||
for i in range(num_tokens)
|
||||
]
|
||||
expected, _, value_scale, score_error = (
|
||||
torch.stack(values) for values in zip(*references)
|
||||
)
|
||||
assert actual.dtype == dtype
|
||||
_assert_sink_output_close(
|
||||
actual,
|
||||
expected,
|
||||
value_scale,
|
||||
score_error,
|
||||
dtype,
|
||||
native=dtype == torch.bfloat16,
|
||||
)
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user