Merge remote-tracking branch 'upstream/main' into patches

Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-03 21:59:03 +07:00
1012 changed files with 61873 additions and 14499 deletions
@@ -1,6 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import gc
import pytest
import torch
@@ -233,6 +235,98 @@ def test_discard_tags():
assert torch.allclose(weights, torch.ones_like(weights))
@pytest.mark.parametrize("tag", ["workspace", None], ids=["workspace", "default"])
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
def test_selective_wake_restores_internal_tags(tag):
"""A selective wake defers only weights/kv_cache; other tags always wake."""
allocator = get_mem_allocator_instance()
with allocator.use_memory_pool(tag):
internal = torch.empty(16 << 20, dtype=torch.uint8, device=DEVICE_TYPE)
with allocator.use_memory_pool("kv_cache"):
kv = torch.empty(32 << 20, dtype=torch.uint8, device=DEVICE_TYPE)
allocator.sleep(offload_tags=())
assert mapped_usage(allocator) == 0
allocator.wake_up(tags=["weights"])
assert mapped_usage(allocator) == internal.nbytes
allocator.wake_up(tags=["kv_cache"])
assert mapped_usage(allocator) == internal.nbytes + kv.nbytes
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
@pytest.mark.skipif(current_platform.is_xpu(), reason="Uses the CuMem allocator")
def test_workspace_scratch_discarded_on_sleep():
"""Workspace scratch lives in the sleep-mode pool: sleep discards it, any
(selective) wake remaps it at the same address so captured graphs replay,
and a buffer outgrown by a resize stays usable until it is released."""
from vllm.device_allocator.sleep_mode_backend import CuMemBackend
from vllm.v1.worker.workspace import WorkspaceManager
allocator = get_mem_allocator_instance()
manager = WorkspaceManager(
torch.device(DEVICE_TYPE),
alloc_context=lambda: allocator.use_memory_pool("workspace"),
)
(small,) = manager.get_simultaneous(((32 << 20,), torch.uint8))
# Resize while a view of the old buffer is alive: the tag is re-entered.
(scratch,) = manager.get_simultaneous(((64 << 20,), torch.uint8))
assert len(allocator.allocator_and_pools["workspace"]) == 2
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
scratch.add_(1)
backend = CuMemBackend()
backend.suspend(level=1)
assert mapped_usage(allocator) == 0
# Discarded, not copied.
assert all(
d.tag == "workspace" and d.cpu_backup_tensor is None
for d in allocator.pointer_to_data.values()
)
backend.resume(tags=["kv_cache"])
assert mapped_usage(allocator) == small.nbytes + scratch.nbytes
scratch.zero_()
graph.replay()
assert int(scratch.sum()) == scratch.numel()
small.fill_(1)
# The next resize releases the outgrown buffer; wake does not remap it.
del small
(big,) = manager.get_simultaneous(((128 << 20,), torch.uint8))
assert mapped_usage(allocator) == scratch.nbytes + big.nbytes
backend.suspend(level=1)
backend.resume(tags=["weights"])
assert mapped_usage(allocator) == scratch.nbytes + big.nbytes
@pytest.mark.parametrize("free_x_early", [True, False], ids=["x-freed", "x-alive"])
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
@pytest.mark.skipif(current_platform.is_xpu(), reason="Uses the CuMem allocator")
def test_reentered_tag(free_x_early):
"""Enter a tag twice (two workspace ubatches): freeing X returns its memory
by the next entry, without touching Y or breaking sleep/wake."""
allocator = get_mem_allocator_instance()
nbytes = 64 << 20
with allocator.use_memory_pool("t"):
xs = [torch.empty(nbytes, dtype=torch.uint8, device=DEVICE_TYPE)]
if free_x_early:
xs.clear()
with allocator.use_memory_pool("t"):
y = torch.empty(nbytes, dtype=torch.uint8, device=DEVICE_TYPE)
xs.clear()
gc.collect()
with allocator.use_memory_pool("t"):
pass
assert mapped_usage(allocator) == nbytes
y.fill_(1)
allocator.sleep(offload_tags="t")
allocator.wake_up()
assert int(y.sum()) == y.numel()
allocator.release_pools()
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
@pytest.mark.skipif(current_platform.is_xpu(), reason="Uses the CuMem allocator")
def test_level2_discards_ordinary_tensor_with_weights_tag():
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
import math
import pytest
import torch
@@ -9,6 +10,7 @@ import torch
import vllm.device_allocator.cumem as cumem
import vllm.envs as envs
from vllm import LLM, AsyncEngineArgs, AsyncLLMEngine, SamplingParams
from vllm.model_executor.model_loader import get_model_loader
from vllm.platforms import current_platform
from vllm.utils.mem_constants import GiB_bytes
@@ -232,6 +234,58 @@ def test_deep_sleep_async(gpu_memory_cleared):
asyncio.run(test())
def _load_weights(worker) -> None:
# Not reload_weights(): its layerwise finalize re-derives the KV scales.
model = worker.model_runner.get_model()
loader = get_model_loader(worker.load_config)
model.load_weights(loader.get_all_weights(worker.model_config, model))
def _attn_kv_scales(model) -> dict[str, list[float]]:
return {
f"{name}.{attr}": getattr(module, attr).flatten().tolist()
for name, module in model.named_modules()
for attr in ("_q_scale", "_k_scale", "_v_scale")
if isinstance(getattr(module, attr, None), torch.Tensor)
}
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Segfaults on ROCm.")
@requires_fp8
@create_new_process_for_each_test()
def test_deep_sleep_compressed_tensors_kv_scales(
monkeypatch: pytest.MonkeyPatch, gpu_memory_cleared
):
"""KV scales must survive level-2 sleep plus a weights reload (else NaN)."""
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
llm = LLM(
"nm-testing/TinyLlama-1.1B-Chat-v1.0-kvcache-fp8-tensor",
enable_sleep_mode=True,
kv_cache_dtype="fp8",
enforce_eager=True,
gpu_memory_utilization=0.6,
)
sampling_params = SamplingParams(temperature=0, max_tokens=10, logprobs=0)
scales = llm.apply_model(_attn_kv_scales)[0]
assert any(v != 1.0 for values in scales.values() for v in values)
expected = llm.generate("How are you?", sampling_params)[0].outputs[0]
llm.sleep(level=2)
llm.wake_up(tags=["weights"])
llm.collective_rpc(_load_weights)
llm.wake_up(tags=["kv_cache"])
assert llm.apply_model(_attn_kv_scales)[0] == scales
actual = llm.generate("How are you?", sampling_params)[0].outputs[0]
assert actual.token_ids == expected.token_ids
logprobs = [
[lp[t].logprob for lp, t in zip(o.logprobs, o.token_ids)]
for o in (expected, actual)
]
assert all(math.isfinite(x) for x in logprobs[1])
assert logprobs[0] == logprobs[1]
@requires_fp8
def test_deep_sleep_fp8_kvcache_mrv1(
monkeypatch: pytest.MonkeyPatch, gpu_memory_cleared
+134 -8
View File
@@ -6,6 +6,7 @@ from pathlib import Path
import pytest
from vllm.benchmarks.sweep import serve as sweep_serve
from vllm.benchmarks.sweep.param_sweep import ParameterSweep, ParameterSweepItem
@@ -115,7 +116,7 @@ class TestParameterSweep:
def test_from_records_list(self):
"""Test creating ParameterSweep from a list of records."""
records = [
records: list[dict[str, object]] = [
{"max_tokens": 100, "temperature": 0.7},
{"max_tokens": 200, "temperature": 0.9},
]
@@ -126,7 +127,7 @@ class TestParameterSweep:
def test_read_from_dict(self):
"""Test creating ParameterSweep from a dict format."""
data = {
data: dict[str, dict[str, object]] = {
"experiment1": {"max_tokens": 100, "temperature": 0.7},
"experiment2": {"max_tokens": 200, "temperature": 0.9},
}
@@ -148,7 +149,7 @@ class TestParameterSweep:
def test_read_json_list_format(self):
"""Test reading JSON file with list format."""
records = [
records: list[dict[str, object]] = [
{"max_tokens": 100, "temperature": 0.7},
{"max_tokens": 200, "temperature": 0.9},
]
@@ -167,7 +168,7 @@ class TestParameterSweep:
def test_read_json_dict_format(self):
"""Test reading JSON file with dict format."""
data = {
data: dict[str, dict[str, object]] = {
"experiment1": {"max_tokens": 100, "temperature": 0.7},
"experiment2": {"max_tokens": 200, "temperature": 0.9},
}
@@ -189,7 +190,7 @@ class TestParameterSweep:
def test_unique_benchmark_names_validation(self):
"""Test that duplicate _benchmark_name values raise an error."""
# Test with duplicate names in list format
records = [
records: list[dict[str, object]] = [
{"_benchmark_name": "exp1", "max_tokens": 100},
{"_benchmark_name": "exp1", "max_tokens": 200},
]
@@ -199,7 +200,7 @@ class TestParameterSweep:
def test_unique_benchmark_names_multiple_duplicates(self):
"""Test validation with multiple duplicate names."""
records = [
records: list[dict[str, object]] = [
{"_benchmark_name": "exp1", "max_tokens": 100},
{"_benchmark_name": "exp1", "max_tokens": 200},
{"_benchmark_name": "exp2", "max_tokens": 300},
@@ -211,7 +212,7 @@ class TestParameterSweep:
def test_no_benchmark_names_allowed(self):
"""Test that records without _benchmark_name are allowed."""
records = [
records: list[dict[str, object]] = [
{"max_tokens": 100, "temperature": 0.7},
{"max_tokens": 200, "temperature": 0.9},
]
@@ -220,7 +221,7 @@ class TestParameterSweep:
def test_mixed_benchmark_names_allowed(self):
"""Test that mixing records with and without _benchmark_name is allowed."""
records = [
records: list[dict[str, object]] = [
{"_benchmark_name": "exp1", "max_tokens": 100},
{"max_tokens": 200, "temperature": 0.9},
]
@@ -247,3 +248,128 @@ class TestParameterSweepItemKeyNormalization:
# The prefix (compilation_config) gets converted to hyphens,
# but the suffix (some_nested_param) is preserved
assert any("compilation-config.some_nested_param" in arg for arg in cmd)
def test_run_comb_excludes_warmup_from_measured_results(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
):
calls: list[tuple[int, int, str]] = []
def fake_run_benchmark(
server,
bench_cmd,
*,
serve_overrides,
bench_overrides,
run_number,
output_path,
dry_run,
):
calls.append(
(run_number, int(bench_overrides["num_prompts"]), output_path.name)
)
return {"run_number": run_number}
monkeypatch.setattr(sweep_serve, "run_benchmark", fake_run_benchmark)
base_path = tmp_path / "combination"
base_path.mkdir()
measured = sweep_serve.run_comb(
None,
[],
serve_comb=ParameterSweepItem(),
bench_comb=ParameterSweepItem({"num_prompts": 320}),
link_vars=[],
base_path=base_path,
num_runs=2,
warmup_num_prompts=32,
dry_run=False,
)
assert calls == [
(-1, 32, "warmup.json"),
(0, 320, "run=0.json"),
(1, 320, "run=1.json"),
]
assert measured == [{"run_number": 0}, {"run_number": 1}]
def test_run_comb_continue_on_error_keeps_later_runs(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
):
calls: list[int] = []
def fake_run_benchmark(
server,
bench_cmd,
*,
serve_overrides,
bench_overrides,
run_number,
output_path,
dry_run,
):
calls.append(run_number)
if run_number == 0:
raise RuntimeError("synthetic run failure")
return {"run_number": run_number}
monkeypatch.setattr(sweep_serve, "run_benchmark", fake_run_benchmark)
base_path = tmp_path / "combination"
base_path.mkdir()
measured = sweep_serve.run_comb(
None,
[],
serve_comb=ParameterSweepItem(),
bench_comb=ParameterSweepItem({"num_prompts": 100}),
link_vars=[],
base_path=base_path,
num_runs=2,
warmup_num_prompts=0,
dry_run=False,
continue_on_error=True,
)
assert calls == [0, 1]
assert measured == [{"run_number": 1}]
assert (base_path / "run=0.failure.json").exists()
assert (base_path / "summary.json").exists()
def test_run_comb_warmup_default_is_backward_compatible(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
):
"""Older callers can omit warmup_num_prompts."""
calls: list[int] = []
def fake_run_benchmark(
server,
bench_cmd,
*,
serve_overrides,
bench_overrides,
run_number,
output_path,
dry_run,
):
calls.append(run_number)
return {"run_number": run_number}
monkeypatch.setattr(sweep_serve, "run_benchmark", fake_run_benchmark)
base_path = tmp_path / "combination"
base_path.mkdir()
measured = sweep_serve.run_comb(
None,
[],
serve_comb=ParameterSweepItem(),
bench_comb=ParameterSweepItem({"num_prompts": 10}),
link_vars=[],
base_path=base_path,
num_runs=1,
dry_run=False,
)
assert calls == [0]
assert measured == [{"run_number": 0}]
+10 -5
View File
@@ -13,6 +13,7 @@ import soundfile as sf
import vllm.benchmarks.datasets.datasets as datasets_module
import vllm.benchmarks.lib.endpoint_request_func as request_func_module
from vllm.benchmarks.lib.endpoint_request_func import RequestFuncInput
from vllm.tokenizers import TokenizerLike
pytestmark = pytest.mark.skip_global_cleanup
@@ -44,6 +45,10 @@ class _CohereNameOnlyTokenizer(_Tokenizer):
super().__init__("cohere/some-local-checkpoint")
def _as_tokenizer(tokenizer: _Tokenizer) -> TokenizerLike:
return cast(TokenizerLike, tokenizer)
def _write_wav(path: Path, duration_s: float = 0.1, sample_rate: int = 16_000) -> None:
num_samples = int(duration_s * sample_rate)
sf.write(path, np.zeros(num_samples, dtype=np.float32), sample_rate)
@@ -109,7 +114,7 @@ def test_asr_dataset_sample_handles_local_audio_paths(tmp_path: Path) -> None:
]
samples = dataset.sample(
tokenizer=_Tokenizer(),
tokenizer=_as_tokenizer(_Tokenizer()),
num_requests=1,
output_len=32,
asr_min_audio_len_sec=0.0,
@@ -146,7 +151,7 @@ def test_asr_dataset_sample_handles_embedded_audio_bytes(
]
samples = dataset.sample(
tokenizer=_Tokenizer(),
tokenizer=_as_tokenizer(_Tokenizer()),
num_requests=1,
output_len=32,
asr_min_audio_len_sec=0.0,
@@ -240,7 +245,7 @@ def _make_asr_dataset(tmp_path: Path) -> datasets_module.ASRDataset:
def test_asr_dataset_cohere_class_name_gets_decoder_prompt(tmp_path: Path) -> None:
dataset = _make_asr_dataset(tmp_path)
samples = dataset.sample(
tokenizer=CohereAsrTokenizer(),
tokenizer=_as_tokenizer(CohereAsrTokenizer()),
num_requests=1,
output_len=32,
asr_min_audio_len_sec=0.0,
@@ -255,7 +260,7 @@ def test_asr_dataset_cohere_name_or_path_fallback_gets_decoder_prompt(
) -> None:
dataset = _make_asr_dataset(tmp_path)
samples = dataset.sample(
tokenizer=_CohereNameOnlyTokenizer(),
tokenizer=_as_tokenizer(_CohereNameOnlyTokenizer()),
num_requests=1,
output_len=32,
asr_min_audio_len_sec=0.0,
@@ -268,7 +273,7 @@ def test_asr_dataset_cohere_name_or_path_fallback_gets_decoder_prompt(
def test_asr_dataset_unknown_tokenizer_gets_empty_prompt(tmp_path: Path) -> None:
dataset = _make_asr_dataset(tmp_path)
samples = dataset.sample(
tokenizer=_Tokenizer(name_or_path="some-other/asr-model"),
tokenizer=_as_tokenizer(_Tokenizer(name_or_path="some-other/asr-model")),
num_requests=1,
output_len=32,
asr_min_audio_len_sec=0.0,
+29 -15
View File
@@ -4,7 +4,7 @@ import json
from argparse import Namespace
from io import BytesIO
from pathlib import Path
from typing import Any
from typing import Any, cast
import pybase64 as base64
import pytest
@@ -17,6 +17,7 @@ from vllm.benchmarks.lib.endpoint_request_func import (
_get_chat_content,
_get_chat_messages,
)
from vllm.tokenizers import TokenizerLike
pytestmark = pytest.mark.skip_global_cleanup
@@ -31,6 +32,10 @@ class _Tokenizer:
return _TokenizedPrompt(prompt)
def _tokenizer() -> TokenizerLike:
return cast(TokenizerLike, _Tokenizer())
def _write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None:
with path.open("w") as f:
for row in rows:
@@ -92,7 +97,7 @@ def test_get_samples_custom_image_cli_path_supports_multi_image_and_content(
],
)
samples = get_samples(_args_for_custom_image(jsonl), _Tokenizer())
samples = get_samples(_args_for_custom_image(jsonl), _tokenizer())
assert len(samples) == 2
assert samples[0].request_id == "req-0"
@@ -106,6 +111,7 @@ def test_get_samples_custom_image_cli_path_supports_multi_image_and_content(
assert samples[1].multi_modal_data is None
assert isinstance(samples[1].prompt, list)
assert samples[1].prompt[0] == {"type": "text", "text": "Now compare "}
assert isinstance(samples[1].prompt[1], dict)
assert samples[1].prompt[1]["image_url"]["url"] == f"file://{image_c}"
@@ -126,7 +132,7 @@ def test_custom_image_dataset_uses_all_image_files(tmp_path: Path) -> None:
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
samples = dataset.sample(
tokenizer=_Tokenizer(),
tokenizer=_tokenizer(),
num_requests=1,
output_len=32,
)
@@ -171,7 +177,7 @@ def test_custom_image_dataset_preserves_interleaved_content_order(
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
samples = dataset.sample(
tokenizer=_Tokenizer(),
tokenizer=_tokenizer(),
num_requests=1,
output_len=32,
)
@@ -181,14 +187,18 @@ def test_custom_image_dataset_preserves_interleaved_content_order(
assert sample.multi_modal_data is None
assert sample.prompt_len == 2
assert isinstance(sample.prompt, list)
assert [part["type"] for part in sample.prompt] == [
parts = []
for part in sample.prompt:
assert isinstance(part, dict)
parts.append(part)
assert [part["type"] for part in parts] == [
"text",
"image_url",
"text",
"image_url",
]
assert sample.prompt[1]["image_url"]["url"] == f"file://{image_a}"
assert sample.prompt[3]["image_url"] == {
assert parts[1]["image_url"]["url"] == f"file://{image_a}"
assert parts[3]["image_url"] == {
"url": f"file://{image_b}",
"detail": "low",
}
@@ -223,7 +233,7 @@ def test_custom_image_dataset_wraps_interleaved_content_for_multimodal_chat(
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
samples = dataset.sample(
tokenizer=_Tokenizer(),
tokenizer=_tokenizer(),
num_requests=1,
output_len=32,
enable_multimodal_chat=True,
@@ -289,7 +299,7 @@ def test_custom_image_dataset_encodes_image_media_when_requested(
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
samples = dataset.sample(
tokenizer=_Tokenizer(),
tokenizer=_tokenizer(),
num_requests=1,
output_len=32,
ensure_client_side_data=True,
@@ -335,7 +345,7 @@ def test_custom_image_dataset_encodes_interleaved_image_media(
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
samples = dataset.sample(
tokenizer=_Tokenizer(),
tokenizer=_tokenizer(),
num_requests=1,
output_len=32,
ensure_client_side_data=True,
@@ -343,9 +353,13 @@ def test_custom_image_dataset_encodes_interleaved_image_media(
sample = samples[0]
assert isinstance(sample.prompt, list)
_assert_png_data_url(sample.prompt[1]["image_url"]["url"])
_assert_png_data_url(sample.prompt[2]["image_url"]["url"])
assert sample.prompt[2]["image_url"]["detail"] == "low"
parts = []
for part in sample.prompt:
assert isinstance(part, dict)
parts.append(part)
_assert_png_data_url(parts[1]["image_url"]["url"])
_assert_png_data_url(parts[2]["image_url"]["url"])
assert parts[2]["image_url"]["detail"] == "low"
@pytest.mark.benchmark
@@ -363,7 +377,7 @@ def test_custom_image_dataset_rejects_invalid_image_media(
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
with pytest.raises(ValueError, match="Invalid image URL"):
dataset.sample(
tokenizer=_Tokenizer(),
tokenizer=_tokenizer(),
num_requests=1,
output_len=32,
ensure_client_side_data=True,
@@ -380,7 +394,7 @@ def test_custom_image_dataset_rejects_invalid_content_part(
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
with pytest.raises(ValueError, match="type 'text', 'image', or 'image_url'"):
dataset.sample(
tokenizer=_Tokenizer(),
tokenizer=_tokenizer(),
num_requests=1,
output_len=32,
)
+2
View File
@@ -37,6 +37,7 @@ def random_dataset_params() -> Params:
def _fingerprint_sample(req: SampleRequest) -> tuple[str, int, int]:
"""Project a SampleRequest into a comparable tuple."""
assert isinstance(req.prompt, str)
return (req.prompt, req.prompt_len, req.expected_output_len)
@@ -153,6 +154,7 @@ def _mm_fingerprint_sample(
- count of multimodal items
- per-item type and URL prefix (e.g., 'data:image/jpeg;base64,')
"""
assert isinstance(req.prompt, str)
items = req.multi_modal_data or []
item_prefixes: list[str] = []
for it in items:
@@ -2,8 +2,11 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
import tempfile
from pathlib import Path
from tempfile import NamedTemporaryFile
from typing import Any, cast
from unittest.mock import Mock
import cv2
import pybase64 as base64
@@ -61,11 +64,17 @@ def test_map_config_to_modality(video_dataset: RandomMultiModalDataset):
@pytest.mark.benchmark
def test_generate_mm_item_video(video_dataset: RandomMultiModalDataset):
"""Test generating multimodal items for video configurations."""
def test_generate_mm_item_video(
video_dataset: RandomMultiModalDataset,
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
):
"""Generate a decodable video without leaving temporary files behind."""
monkeypatch.setattr(tempfile, "tempdir", str(tmp_path))
# Test video item generation
video_config = (64, 48, 8) # height, width, num_frames
result = video_dataset.generate_mm_item(video_config)
assert not list(tmp_path.iterdir())
# Check the result structure matches OpenAI API format
assert isinstance(result, dict)
@@ -105,6 +114,33 @@ def test_generate_mm_item_video(video_dataset: RandomMultiModalDataset):
os.unlink(temp_path)
@pytest.mark.benchmark
@pytest.mark.parametrize("writer_opened", [False, True], ids=["open", "write"])
def test_generate_synthetic_video_cleans_up_on_error(
video_dataset: RandomMultiModalDataset,
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
writer_opened: bool,
):
"""Release the writer and remove temporary storage when encoding fails."""
monkeypatch.setattr(tempfile, "tempdir", str(tmp_path))
writer = Mock(spec=cv2.VideoWriter)
writer.isOpened.return_value = writer_opened
writer.write.side_effect = RuntimeError("Failed to write video frame")
monkeypatch.setattr(cv2, "VideoWriter", lambda *args, **kwargs: writer)
error = (
"Failed to write video frame"
if writer_opened
else "Failed to create video writer"
)
with pytest.raises(RuntimeError, match=error):
video_dataset.generate_synthetic_video(48, 64, 8)
assert not list(tmp_path.iterdir())
writer.release.assert_called_once_with()
@pytest.mark.benchmark
def test_generate_mm_item_image(video_dataset: RandomMultiModalDataset):
"""Test generating multimodal items for image configurations."""
+4 -4
View File
@@ -19,7 +19,7 @@ class _FakeTokenizer(TokenizerLike):
# -- Properties required by TokenizerLike --
@classmethod
def from_pretrained(cls, path_or_repo_id, *a, **kw): # type: ignore[override]
def from_pretrained(cls, path_or_repo_id, *a, **kw):
return cls()
@property
@@ -80,16 +80,16 @@ class _FakeTokenizer(TokenizerLike):
def apply_chat_template(self, messages, **kw): # type: ignore[override]
raise NotImplementedError
def convert_tokens_to_ids(self, tokens): # type: ignore[override]
def convert_tokens_to_ids(self, tokens):
raise NotImplementedError
def convert_tokens_to_string(self, tokens: list[str]) -> str:
raise NotImplementedError
def decode(self, ids, skip_special_tokens: bool = False) -> str: # type: ignore[override]
def decode(self, ids, skip_special_tokens: bool = False) -> str:
raise NotImplementedError
def convert_ids_to_tokens( # type: ignore[override]
def convert_ids_to_tokens(
self, ids, skip_special_tokens: bool = False
) -> list[str]:
raise NotImplementedError
+3 -2
View File
@@ -9,6 +9,7 @@ serve-side dataset coverage.
"""
import subprocess
from argparse import Namespace
from types import SimpleNamespace
import pytest
@@ -289,7 +290,7 @@ def test_to_serve_args_supports_mm_processor_flags() -> None:
explicit 0 instead of coercing it to None (which later crashed
RandomMultiModalDataset.get_prefix on ``None <= 0``).
"""
args = SimpleNamespace(
args = Namespace(
random_input_len=32,
random_output_len=1,
random_prefix_len=0,
@@ -306,7 +307,7 @@ def test_to_serve_args_supports_mm_processor_flags() -> None:
def test_to_serve_args_keeps_legacy_flag_fallback() -> None:
"""Callers that only set the legacy flags still get them mapped."""
args = SimpleNamespace(
args = Namespace(
input_len=256,
output_len=8,
prefix_len=16,
+45 -16
View File
@@ -3,6 +3,7 @@
import json
import logging
from contextlib import nullcontext
import pytest
@@ -174,13 +175,18 @@ def test_async_tp_pass_correctness(
@create_new_process_for_each_test()
def test_async_tp_pass_nvfp4_correctness(num_gpus_available: int):
if (
not current_platform.is_cuda()
or not current_platform.is_device_capability_family(100)
):
pytest.skip("NVFP4 requires Blackwell")
if not has_flashinfer():
def test_async_tp_pass_nvfp4_correctness(num_gpus_available: int, caplog_mp_spawn):
is_blackwell = current_platform.is_cuda() and (
current_platform.is_device_capability_family(100)
)
is_rocm_cdna = False
if current_platform.is_rocm():
from vllm.platforms.rocm import on_cdna
is_rocm_cdna = on_cdna()
if not (is_blackwell or is_rocm_cdna):
pytest.skip("NVFP4 AsyncTP requires Blackwell or ROCm CDNA")
if is_blackwell and not has_flashinfer():
pytest.skip("FlashInfer is required for the NVFP4 AsyncTP path")
tp_size = 2
@@ -196,8 +202,7 @@ def test_async_tp_pass_nvfp4_correctness(num_gpus_available: int):
"8",
"--load-format",
"dummy",
"--linear-backend",
"flashinfer_cutlass",
*(["--linear-backend", "flashinfer_cutlass"] if is_blackwell else []),
"--hf-overrides",
json.dumps(NVFP4_HF_OVERRIDES),
]
@@ -213,6 +218,8 @@ def test_async_tp_pass_nvfp4_correctness(num_gpus_available: int):
"sp_min_token_num": 1,
},
}
if is_rocm_cdna:
compilation_config["inductor_compile_config"] = {"force_disable_caches": True}
async_tp_args = [
*common_args,
@@ -232,10 +239,32 @@ def test_async_tp_pass_nvfp4_correctness(num_gpus_available: int):
"mp",
]
compare_two_settings(
NVFP4_MODEL_ID,
async_tp_args,
tp_args,
method="generate",
force_v1_runner=True,
)
env = {"VLLM_DISABLE_COMPILE_CACHE": "1"} if is_rocm_cdna else None
log_context = caplog_mp_spawn(logging.DEBUG) if is_rocm_cdna else nullcontext()
with log_context as logs:
compare_two_settings(
NVFP4_MODEL_ID,
async_tp_args,
tp_args,
env1=env,
env2=env,
method="generate",
force_v1_runner=True,
)
if is_rocm_cdna:
assert logs is not None
matches_by_rank = {
rank: [
int(count)
for line in logs.text.splitlines()
if line.startswith(f"(Worker_TP{rank} pid=")
for count in FUSION_LOG_PATTERNS["async_tp"].findall(line)
]
for rank in range(tp_size)
}
assert all(
any(count > 0 for count in matches) for matches in matches_by_rank.values()
), (
"Expected positive NVFP4 AsyncTP rewrites on both TP workers, "
f"found {matches_by_rank}"
)
@@ -374,11 +374,16 @@ def test_tp_sp_generation_prompt_embeds(
@create_new_process_for_each_test()
def test_tp_sp_nvfp4_generation(num_gpus_available: int):
if (
not current_platform.is_cuda()
or not current_platform.is_device_capability_family(100)
):
pytest.skip("NVFP4 requires Blackwell")
is_blackwell = current_platform.is_cuda() and (
current_platform.is_device_capability_family(100)
)
is_rocm_cdna = False
if current_platform.is_rocm():
from vllm.platforms.rocm import on_cdna
is_rocm_cdna = on_cdna()
if not (is_blackwell or is_rocm_cdna):
pytest.skip("NVFP4 sequence parallelism requires Blackwell or ROCm CDNA")
comparison = _build_sp_args(
NVFP4_MODEL_ID,
@@ -1,10 +1,13 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from functools import partial
import pytest
from tests.models.utils import dummy_hf_overrides
from vllm.compilation.counter import compilation_counter
from vllm.config import VllmConfig
from vllm.config.compilation import CompilationMode
from vllm.config.compilation import CompilationMode, CUDAGraphMode
from vllm.platforms import current_platform
@@ -14,6 +17,77 @@ def test_compile():
assert not vllm_config.compilation_config.compile_mm_encoder
@pytest.mark.forked
@pytest.mark.core_model
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Skip if not cuda")
@pytest.mark.parametrize(
("model", "tokenizer", "config_format", "tokenizer_mode", "model_arch"),
[
pytest.param(
"mistralai/Pixtral-12B-2409",
None,
"mistral",
"mistral",
"PixtralForConditionalGeneration",
id="pixtral",
),
pytest.param(
"mistralai/Mistral-Small-3.1-24B-Instruct-2503",
"mistral-community/pixtral-12b",
"hf",
"auto",
"Mistral3ForConditionalGeneration",
id="mistral3-hf",
),
],
)
@pytest.mark.parametrize(
("compile_mm_encoder", "expected_models"),
[
# The dummy model has two compiled text components. Enabling encoder
# compilation adds its single vision transformer block.
pytest.param(False, 2, id="disabled"),
pytest.param(True, 3, id="enabled"),
],
)
def test_pixtral_compilation(
vllm_runner,
monkeypatch,
model: str,
tokenizer: str | None,
config_format: str,
tokenizer_mode: str,
model_arch: str,
compile_mm_encoder: bool,
expected_models: int,
):
"""Test compilation for both Pixtral vision encoder implementations."""
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
monkeypatch.setenv("VLLM_USE_FLASHINFER_SAMPLER", "0")
with (
compilation_counter.expect(num_models_seen=expected_models),
vllm_runner(
model,
tokenizer_name=tokenizer,
tokenizer_mode=tokenizer_mode,
config_format=config_format,
load_format="dummy",
hf_overrides=partial(dummy_hf_overrides, model_arch=model_arch),
max_model_len=4096,
limit_mm_per_prompt={"image": 1},
gpu_memory_utilization=0.8,
attention_backend="FLASH_ATTN",
compilation_config={
"mode": CompilationMode.VLLM_COMPILE,
"compile_mm_encoder": compile_mm_encoder,
"cudagraph_mode": CUDAGraphMode.NONE,
},
) as _,
):
pass
# forked needed to workaround https://github.com/vllm-project/vllm/issues/21073
@pytest.mark.forked
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Skip if not cuda")
+3 -1
View File
@@ -111,7 +111,9 @@ def test_tp2_ar_rms_fp8_fusions(
model_kwargs["max_model_len"] = 1024
model_kwargs["kernel_config"] = {"enable_flashinfer_autotune": False}
model_kwargs["disable_custom_all_reduce"] = False
if current_platform.is_rocm() and "llama-3" in model_name_lower:
if current_platform.is_rocm() and (
"llama-3" in model_name_lower or "qwen" in model_name_lower
):
model_kwargs.update(
gpu_memory_utilization=0.01,
kv_cache_memory_bytes=64 * 1024**2,
@@ -0,0 +1,60 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""ROCm MoE coverage for the TP2 GEMM/collective fusion contract."""
import pytest
from tests.utils import multi_gpu_test
from vllm.config import PassConfig
from vllm.platforms import current_platform
from .common import INDUCTOR_GRAPH_PARTITION
from .models import TRITON_ATTN, qwen3_a3b
@multi_gpu_test(num_gpus=2)
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm AsyncTP contract")
@pytest.mark.parametrize("inductor_graph_partition", INDUCTOR_GRAPH_PARTITION[:1])
@pytest.mark.parametrize("attn_backend", [TRITON_ATTN])
def test_tp2_rocm_moe_async_tp_bf16_fusions(
inductor_graph_partition, attn_backend, run_e2e_fusion_test
):
"""MoE dense projections retain the Qwen SP/AsyncTP rewrite counts."""
n_layers = 4
run_e2e_fusion_test(
qwen3_a3b.model_name,
qwen3_a3b.matches(n_layers),
dict(
hf_overrides=dict(
num_hidden_layers=n_layers,
hidden_size=512,
intermediate_size=1024,
moe_intermediate_size=128,
num_attention_heads=8,
num_key_value_heads=2,
head_dim=64,
num_experts=8,
num_experts_per_tok=2,
),
load_format="dummy",
max_model_len=1024,
kv_cache_memory_bytes=64 * 1024**2,
gpu_memory_utilization=0.01,
max_num_batched_tokens=1024,
max_num_seqs=32,
),
attn_backend,
dict(
use_inductor_graph_partition=inductor_graph_partition,
custom_ops=["-rms_norm"],
pass_config=PassConfig(
enable_qk_norm_rope_fusion=True,
enable_sp=True,
fuse_gemm_comms=True,
fuse_allreduce_rms=False,
sp_min_token_num=512,
),
),
["norm_rope_fusion", "sequence_parallel", "async_tp"],
tp_size=2,
)
-4
View File
@@ -60,10 +60,6 @@ from vllm.utils.deep_gemm import (
is_deep_gemm_supported,
)
FP8_DTYPE = current_platform.fp8_dtype()
RMS_ADD_OP = torch.ops._C.fused_add_rms_norm.default
# Kernel and group_shape combinations: (kernel, group_shape)
# CUDA kernels
CUDA_KERNEL_GROUPSHAPE_COMBINATIONS = [
+3 -2
View File
@@ -33,6 +33,7 @@ from vllm.config import (
VllmConfig,
set_current_vllm_config,
)
from vllm.config.cache import CacheDType
from vllm.forward_context import get_forward_context, set_forward_context
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.quantization.utils.quant_utils import (
@@ -198,6 +199,7 @@ class TestAttentionAiterFp8StaticQuantPatternModel(
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.fp8_linear.kernel.quant_fp8.use_aiter = True
assert self.fp8_linear.input_scale is not None
self.fp8_linear.input_scale.fill_(0.125)
@@ -246,7 +248,6 @@ class TestAttentionNvfp4QuantPatternModel(AttentionQuantPatternModel):
PATTERN_TEST_MODELS_FP8: list[tuple[str, type]] = []
PATTERN_TEST_MODELS_FP4: list[tuple[str, type]] = []
HEADS: list[tuple[int, int]] = []
SPLIT_ATTENTION: list[bool] = []
BACKENDS_FP8: list[AttentionBackendEnum] = []
BACKENDS_FP4: list[AttentionBackendEnum] = []
@@ -307,7 +308,7 @@ def test_attention_quant_pattern(
head_size: int,
batch_size: int,
dtype: torch.dtype,
kv_cache_dtype: str,
kv_cache_dtype: CacheDType,
custom_ops: str,
model_name: str,
model_class: type[AttentionQuantPatternModel],
@@ -24,6 +24,7 @@ from vllm.config import (
PassConfig,
VllmConfig,
)
from vllm.config.cache import CacheDType
from vllm.forward_context import get_forward_context, set_forward_context
from vllm.model_executor.layers.attention import MLAAttention
from vllm.model_executor.layers.linear import ColumnParallelLinear
@@ -254,7 +255,7 @@ def test_mla_rope_kvcache_cat_fusion(
block_size: int,
is_neox: bool,
dtype: torch.dtype,
kv_cache_dtype: str,
kv_cache_dtype: CacheDType,
monkeypatch: pytest.MonkeyPatch,
):
torch.set_default_device("cuda")
+1 -1
View File
@@ -28,7 +28,7 @@ def test_bad_callable():
pass_manager.configure(config)
with pytest.raises(AssertionError):
pass_manager.add(simple_callable) # type: ignore[arg-type]
pass_manager.add(simple_callable)
# Pass that inherits from InductorPass
@@ -39,6 +39,7 @@ from vllm.config import (
PassConfig,
VllmConfig,
)
from vllm.config.cache import CacheDType
from vllm.forward_context import get_forward_context, set_forward_context
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -342,7 +343,7 @@ def _run_qk_norm_rope_kvcache_fusion_test(
use_shuffle_kv_layout: str,
kv_layout: KVCacheLayout,
dtype: torch.dtype,
kv_cache_dtype: str,
kv_cache_dtype: CacheDType,
rms_norm_eps: float,
custom_op: str,
monkeypatch: pytest.MonkeyPatch,
@@ -611,7 +612,7 @@ def test_qk_norm_rope_kvcache_fusion(
kv_layout: KVCacheLayout,
block_size: int,
dtype: torch.dtype,
kv_cache_dtype: str,
kv_cache_dtype: CacheDType,
rms_norm_eps: float,
custom_op: str,
monkeypatch: pytest.MonkeyPatch,
@@ -663,7 +664,7 @@ def test_qk_norm_mrope_kvcache_fusion(
mrope_interleaved: bool,
is_neox: bool,
kv_layout: KVCacheLayout,
kv_cache_dtype: str,
kv_cache_dtype: CacheDType,
monkeypatch: pytest.MonkeyPatch,
):
_run_qk_norm_rope_kvcache_fusion_test(
@@ -30,6 +30,7 @@ from vllm.config import (
PassConfig,
VllmConfig,
)
from vllm.config.cache import CacheDType
from vllm.config.utils import Range
from vllm.forward_context import get_forward_context, set_forward_context
from vllm.model_executor.layers.attention import Attention
@@ -324,7 +325,7 @@ def test_rope_kvcache_fusion(
block_size: int,
is_neox: bool,
dtype: torch.dtype,
kv_cache_dtype: str,
kv_cache_dtype: CacheDType,
monkeypatch: pytest.MonkeyPatch,
):
torch.set_default_device("cuda")
@@ -468,7 +469,7 @@ def test_rope_static_qquant_kvcache_fusion(
block_size: int,
is_neox: bool,
dtype: torch.dtype,
kv_cache_dtype: str,
kv_cache_dtype: CacheDType,
monkeypatch: pytest.MonkeyPatch,
):
torch.set_default_device("cuda")
@@ -60,7 +60,7 @@ class TestSiluMulFp8QuantModel(torch.nn.Module):
def __init__(
self,
hidden_size: int,
force_kernel: FP8ScaledMMLinearKernel,
force_kernel: type[FP8ScaledMMLinearKernel],
dtype: torch.dtype,
**kwargs,
):
@@ -282,7 +282,7 @@ def test_fusion_silu_and_mul_quant(
],
enable_silu_mul_custom_op: bool,
enable_quant_fp8_custom_op: bool,
force_kernel: FP8ScaledMMLinearKernel | None,
force_kernel: type[FP8ScaledMMLinearKernel] | None,
monkeypatch: pytest.MonkeyPatch,
):
if model_class is TestSiluMulNvfp4QuantModel and not is_nvfp4_supported():
@@ -330,7 +330,10 @@ def test_fusion_silu_and_mul_quant(
passes = [NoOpEliminationPass(config), *fusion_passes, PostCleanupPass(config)]
backend = TestBackend(*passes)
model = model_class(
hidden_size=hidden_size, force_kernel=force_kernel, x=x, dtype=dtype
hidden_size=hidden_size,
force_kernel=force_kernel,
x=x,
dtype=dtype,
)
# First dimension dynamic
@@ -84,7 +84,7 @@ def test_manual_fusion_fp8_static_with_linear(
num_tokens: int,
hidden_size: int,
dtype: torch.dtype,
force_kernel: FP8ScaledMMLinearKernel,
force_kernel: type[FP8ScaledMMLinearKernel],
):
"""Test manual fusion with real FP8 linear layer (kFp8StaticTensorSym).
+48
View File
@@ -0,0 +1,48 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from pathlib import Path
from unittest.mock import MagicMock, patch
import pytest
from vllm.compilation.compiler_interface import InductorStandaloneAdaptor
@pytest.mark.parametrize("save_format", ["binary", "unpacked"])
def test_inductor_standalone_load_uses_current_cache_dir(
tmp_path: Path,
save_format: str,
):
old_cache_dir = tmp_path / "old"
new_cache_dir = tmp_path / "new"
key = "artifact_shape_None_subgraph_0"
adaptor = InductorStandaloneAdaptor(save_format=save_format)
adaptor.initialize_cache(str(new_cache_dir))
# Simulate a handle persisted before the cache directory was relocated.
handle = (key, str(old_cache_dir / key))
with (
patch(
"torch._inductor.CompiledArtifact.load",
return_value=MagicMock(),
) as load_mock,
patch(
"torch._inductor.compile_fx.graph_returns_tuple",
return_value=True,
),
):
adaptor.load(
handle=handle,
graph=MagicMock(),
example_inputs=[],
graph_index=0,
compile_range=MagicMock(),
)
load_mock.assert_called_once_with(
path=str(new_cache_dir / key),
format=save_format,
)
+2 -2
View File
@@ -133,7 +133,7 @@ def test_get_diff_sampling_param_includes_penalties():
},
)
diff = ModelConfig.get_diff_sampling_param(fake) # type: ignore[arg-type]
diff = ModelConfig.get_diff_sampling_param(fake)
assert diff["presence_penalty"] == 1.5
assert diff["frequency_penalty"] == 0.5
@@ -161,7 +161,7 @@ def test_generation_config_penalties_survive_diff_sampling_filter():
},
)
diff = ModelConfig.get_diff_sampling_param(fake) # type: ignore[arg-type]
diff = ModelConfig.get_diff_sampling_param(fake)
assert diff["presence_penalty"] == 1.1
assert diff["frequency_penalty"] == 0.7
+61 -2
View File
@@ -8,7 +8,12 @@ import pytest
from vllm.config.cache import CacheConfig
from vllm.config.scheduler import SchedulerConfig
from vllm.config.utils import get_hash_factors, hash_factors, normalize_value
from vllm.config.utils import (
get_attr_docs,
get_hash_factors,
hash_factors,
normalize_value,
)
# Helpers
@@ -114,7 +119,7 @@ def test_normalize_value_uuid_and_to_json():
[
(lambda x: x),
(type("CallableInstance", (), {"__call__": lambda self: 0}))(),
(lambda: (lambda: 0))(), # nested function instance
(lambda: lambda: 0)(), # nested function instance
],
)
def test_error_cases(bad):
@@ -168,6 +173,53 @@ def test_classes_are_types():
assert endswith_fqname(LocalDummy, ".LocalDummy")
def test_ir_config_hash_initializes_provider_and_tracks_implementation(monkeypatch):
from types import SimpleNamespace
from vllm.config.kernel import IrOpPriorityConfig
from vllm.ir.op import IrOp
from vllm.platforms import current_platform
op = IrOp.registry["rms_norm"]
implementation = SimpleNamespace(uuid=lambda: "first-implementation")
def import_ir_kernels():
monkeypatch.setitem(op.impls, "test_lazy_provider", implementation)
monkeypatch.setattr(current_platform, "import_ir_kernels", import_ir_kernels)
config = IrOpPriorityConfig(rms_norm=["test_lazy_provider"])
first_hash = config.compute_hash()
assert config.compute_hash() == first_hash
implementation.uuid = lambda: "changed-implementation"
assert config.compute_hash() != first_hash
def test_get_attr_docs_includes_inherited_fields():
"""get_attr_docs must collect attribute docstrings inherited from base
classes, not just those declared in the class's own body."""
@dataclass
class Base:
inherited: object = None
"""doc for inherited (base)."""
overridden: object = None
"""base version of overridden."""
@dataclass
class Derived(Base):
own: object = None
"""doc for own (derived)."""
overridden: object = None
"""derived override of overridden."""
docs = get_attr_docs(Derived)
assert docs["own"] == "doc for own (derived)."
assert docs["inherited"] == "doc for inherited (base)."
assert docs["overridden"] == "derived override of overridden."
def test_envs_compile_factors_stable():
"""Test that envs.compile_factors() hash is stable across fresh initializations.
@@ -303,3 +355,10 @@ print(hash_factors(envs.compile_factors()))
"HOME relocation changed the compile-cache env hash - a "
"location-only derived var is leaking into the key"
)
def test_get_attr_docs_rejects_non_class():
"""The annotation forbids it, but the runtime guard still protects
untyped callers, so keep asserting the documented TypeError."""
with pytest.raises(TypeError, match="Given object was not a class."):
get_attr_docs(object()) # type: ignore[arg-type]
+42 -5
View File
@@ -40,18 +40,18 @@ def test_supports_multimodal_inputs(model_id, limit_mm_per_prompt, expected):
def test_mm_encoder_attn_backend_str_conversion():
config = MultiModalConfig(mm_encoder_attn_backend="FLASH_ATTN") # type: ignore[arg-type]
config = MultiModalConfig(mm_encoder_attn_backend="FLASH_ATTN")
assert config.mm_encoder_attn_backend == AttentionBackendEnum.FLASH_ATTN
def test_mm_encoder_attn_backend_invalid():
with pytest.raises(ValueError):
MultiModalConfig(mm_encoder_attn_backend="not_a_backend") # type: ignore[arg-type]
MultiModalConfig(mm_encoder_attn_backend="not_a_backend")
def test_mm_hasher_algorithm_invalid():
with pytest.raises(ValueError, match="mm_hasher_algorithm"):
MultiModalConfig(mm_hasher_algorithm="md5") # type: ignore[arg-type]
MultiModalConfig(mm_hasher_algorithm="md5")
def test_mm_encoder_attn_backend_hash_updates():
@@ -240,6 +240,43 @@ def test_merge_mm_processor_kwargs_scoped_precedence():
}
def test_merge_mm_processor_kwargs_preserves_scoped_siblings_and_precedence():
config = MultiModalConfig(
mm_processor_kwargs={
"size": {
"shortest_edge": 100,
"longest_edge": 1000,
},
"videos_kwargs": {
"fps": 2,
"size": {"longest_edge": 1200},
},
},
mm_device_do_normalize=False,
)
assert config.merge_mm_processor_kwargs(
{
"size": {"longest_edge": 1800},
"videos_kwargs": {
"size": {"shortest_edge": 200},
},
}
) == {
"size": {
"shortest_edge": 100,
"longest_edge": 1800,
},
"videos_kwargs": {
"fps": 2,
"size": {
"shortest_edge": 200,
"longest_edge": 1800,
},
},
}
def test_merge_mm_processor_kwargs_mapping_vs_replacement():
config = MultiModalConfig(
mm_processor_kwargs={
@@ -523,7 +560,7 @@ def _resolve_mm_processor_device(
"""Run the `auto` resolution and report where the processor ended up."""
mm_config = MultiModalConfig(
mm_processor_kwargs={} if device is None else {"device": device},
mm_tensor_ipc=mm_tensor_ipc, # type: ignore[arg-type]
mm_tensor_ipc=mm_tensor_ipc,
)
model_config = MagicMock(spec=ModelConfig)
model_config.multimodal_config = mm_config
@@ -625,7 +662,7 @@ def _resolve_mm_video_decode_device(
the resulting video media IO kwargs."""
mm_config = MultiModalConfig(
mm_processor_kwargs={} if device is None else {"device": device},
mm_tensor_ipc=mm_tensor_ipc, # type: ignore[arg-type]
mm_tensor_ipc=mm_tensor_ipc,
media_io_kwargs={} if video_kwargs is None else {"video": dict(video_kwargs)},
)
model_config = MagicMock(spec=ModelConfig)
+53
View File
@@ -0,0 +1,53 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for ObservabilityConfig validation."""
import pytest
from pydantic import ValidationError
from vllm.config.observability import ObservabilityConfig
def test_custom_histogram_buckets_default_none():
assert ObservabilityConfig().custom_histogram_buckets is None
def test_custom_histogram_buckets_valid():
config = ObservabilityConfig(
custom_histogram_buckets={
"request_latency": [0.01, 0.05, 0.1, 0.5],
"time_to_first_token": [1, 2, 5],
}
)
assert config.custom_histogram_buckets == {
"request_latency": [0.01, 0.05, 0.1, 0.5],
"time_to_first_token": [1.0, 2.0, 5.0],
}
@pytest.mark.parametrize(
("buckets", "match"),
[
({"bogus": [1.0, 2.0]}, "unknown bucket family 'bogus'"),
({"request_latency": []}, "must not be empty"),
({"request_latency": [0.0, 1.0]}, "must be finite and greater than 0"),
({"request_latency": [-1.0, 1.0]}, "must be finite and greater than 0"),
(
{"request_latency": [1.0, float("inf")]},
"must be finite and greater than 0",
),
({"request_latency": [float("nan")]}, "must be finite and greater than 0"),
({"request_latency": [1.0, 1.0]}, "must be strictly increasing"),
({"request_latency": [2.0, 1.0]}, "must be strictly increasing"),
({"request_latency": [True, 2.0]}, "must be a number, not a boolean"),
({"request_latency": [False]}, "must be a number, not a boolean"),
(
{"request_latency": [1.0, 2.0], "request_tokens": [16, 8]},
r"request_tokens.*must be strictly increasing",
),
],
)
def test_custom_histogram_buckets_rejects_invalid(buckets, match):
"""Each validation rule must name the field, the value, and the rule."""
with pytest.raises(ValidationError, match=match):
ObservabilityConfig(custom_histogram_buckets=buckets)
+1 -6
View File
@@ -317,11 +317,6 @@ def example_system_message() -> str:
return f.read()
@pytest.fixture
def example_long_prompts() -> list[str]:
return [prompt for filename in _LONG_PROMPTS for prompt in _read_prompts(filename)]
@pytest.fixture(scope="session")
def image_assets() -> ImageTestAssets:
return IMAGE_ASSETS
@@ -1168,7 +1163,7 @@ class VllmRunner:
output_logprobs = sample.logprobs
if include_prompt_token_ids:
outputs.append(
( # type: ignore[arg-type]
(
output_ids,
output_str,
output_logprobs,
@@ -205,6 +205,42 @@ def test_custom_allreduce_filters_dtype(
assert communicator.should_custom_ar(torch.empty(16, dtype=dtype)) is expected
@pytest.mark.parametrize("batch_invariant", [False, True])
def test_custom_allreduce_size_gate_ignored_under_batch_invariance(
batch_invariant: bool,
) -> None:
"""Batch invariance must not switch backends based on the input size."""
communicator = car.CustomAllreduce.__new__(car.CustomAllreduce)
communicator.disabled = False
communicator.world_size = 2
communicator.max_size = 1024
communicator.batch_invariant = batch_invariant
communicator._ptr = 0
oversized = torch.empty(1024, dtype=torch.float16)
assert communicator.should_custom_ar(oversized) is batch_invariant
@pytest.mark.parametrize("batch_invariant", [False, True])
def test_custom_reduce_scatter_disabled_under_batch_invariance(
monkeypatch, batch_invariant: bool
) -> None:
"""Reduce-scatter stays on one backend under batch invariance."""
monkeypatch.setattr(car.current_platform, "is_cuda", lambda: True)
communicator = car.CustomAllreduce.__new__(car.CustomAllreduce)
communicator.disabled = False
communicator.world_size = 2
communicator.fully_connected = True
communicator.mnnvl_only = False
communicator.mnnvl_multicast_ptr = 0
communicator.max_reduce_scatter_size = 1024
communicator.max_mnnvl_reduce_scatter_size = 1024
communicator.batch_invariant = batch_invariant
in_range = torch.empty(64, dtype=torch.float16)
assert communicator.should_custom_reduce_scatter(in_range) is not batch_invariant
@pytest.mark.parametrize(
("major", "local_multicast", "expected"),
[
@@ -622,6 +658,60 @@ def eager_allreduce(
torch.testing.assert_close(out, inp * (tp_size**num_communication))
@ray.remote(num_gpus=1, max_calls=1)
def chunked_allreduce(
monkeypatch: pytest.MonkeyPatch,
tp_size,
pp_size,
rank,
distributed_init_port,
):
"""Inputs above max_size are reduced in chunks and match the fixed-order
reference bitwise, both eagerly and inside a CUDA graph."""
with monkeypatch.context() as m:
m.delenv("CUDA_VISIBLE_DEVICES", raising=False)
m.delenv("HIP_VISIBLE_DEVICES", raising=False)
m.setenv("VLLM_CUSTOM_ALLREDUCE_ALGO", "1stage")
device = torch.device(f"cuda:{rank}")
torch.accelerator.set_device_index(device)
init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port)
ensure_model_parallel_initialized(tp_size, pp_size)
group = get_tp_group().device_group
fa = get_tp_group().device_communicator.ca_comm
# Two full chunks plus a partial one.
dtype = torch.bfloat16
chunk_numel = fa.max_size // dtype.itemsize
inp = torch.randn(2 * chunk_numel + 4096, device=device).to(dtype)
gathered = [torch.empty_like(inp) for _ in range(tp_size)]
dist.all_gather(gathered, inp, group=group)
ref = gathered[0].float()
for peer in gathered[1:]:
ref = ref + peer.float()
ref = ref.to(torch.bfloat16)
out = fa.all_reduce(inp, registered=False)
assert torch.equal(out, ref)
# Weak-contiguous but not C-contiguous: a transposed matrix.
inp_t, ref_t = inp.view(-1, 4096).t(), ref.view(-1, 4096).t()
assert torch.equal(fa.all_reduce(inp_t, registered=False), ref_t)
with fa.capture():
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
out = fa.all_reduce(inp, registered=True)
graph.replay()
assert torch.equal(out, ref)
@pytest.mark.parametrize("tp_size", [2, 4])
def test_custom_allreduce_chunked(monkeypatch: pytest.MonkeyPatch, tp_size):
if tp_size > torch.accelerator.device_count():
pytest.skip("Not enough GPUs to run the test.")
multi_process_parallel(monkeypatch, tp_size, 1, chunked_allreduce)
@pytest.mark.parametrize("tp_size", [2])
@pytest.mark.parametrize("pipeline_parallel_size", [1, 2])
@pytest.mark.parametrize("test_target", [eager_allreduce, graph_allreduce])
+40 -20
View File
@@ -39,10 +39,10 @@ from vllm.distributed.parallel_state import (
initialize_model_parallel,
)
from vllm.forward_context import set_forward_context
from vllm.models.deepseek_v41.common.engram import EngramLayout
from vllm.models.deepseek_v41.nvidia import engram as engram_ops
from vllm.models.deepseek_v41.nvidia.engram import (
from vllm.models.deepseek_v41.common import engram as engram_ops
from vllm.models.deepseek_v41.common.engram import (
Engram,
EngramLayout,
ParallelEngramEmbedding,
engram_head_shard_rank,
gather_engram_hashes,
@@ -507,20 +507,22 @@ def _check_dummy_hash_model_forward(
assert torch.all(hashes == engram_ops.DEAD_ID)
output = engram.embed(hashes[:, 0])
# Trailing None is previous_aux; this stub captures no aux states.
return output, None, None, None, None, None
return output, hidden, hidden, hidden, hidden, None
model = SimpleNamespace(
use_mega_moe=False,
use_sequence_parallel=False,
fuse_mhc_all_reduce=False,
engram_hash=state,
engram_swa_prefix="swa",
engram_dp_shared_memory=dp_shared_memory,
layers=[Decoder(engram=engram)],
start_layer=0,
end_layer=1,
aux_hidden_state_layers=(),
)
model = model_ops.DeepseekV4Model.__new__(model_ops.DeepseekV4Model)
torch.nn.Module.__init__(model)
model.use_mega_moe = False
model.use_sequence_parallel = False
model.fuse_mhc_all_reduce = False
model.engram_hash = state
model.engram_swa_prefix = "swa"
model.engram_dp_shared_memory = dp_shared_memory
model.layers = [Decoder(engram=engram)]
model.start_layer = 0
model.end_layer = 1
model.decoder_replay_start = model.end_layer
model.decoder_replay_layers = None
model.aux_hidden_state_layers = ()
metadata = (
None
if dp_rank == 1
@@ -550,8 +552,7 @@ def _check_dummy_hash_model_forward(
dtype=torch.int32,
),
):
output = model_ops.DeepseekV4Model.forward(
model,
output = model(
torch.arange(tokens, device="cuda"),
torch.arange(tokens, device="cuda"),
intermediate_tensors=None,
@@ -685,11 +686,30 @@ def test_engram_dp_shared_memory_runtime_requirements(
)
def test_engram_tables_too_large_for_shm_are_not_shared(monkeypatch):
"""A /dev/shm smaller than the tables must fall back instead of failing startup."""
@pytest.mark.parametrize("query_fails", [False, True])
def test_engram_tables_too_large_for_shm_are_not_shared(monkeypatch, query_fails):
"""A /dev/shm that is too small or cannot be queried must fall back, and the
leader must still broadcast that decision so its peers do not hang."""
broadcasts = []
def broadcast(obj):
broadcasts.append(obj)
return obj
def query(*args, **kwargs):
raise OSError("statvfs failed")
monkeypatch.setattr(engram_ops, "get_engram_dp_size", lambda: 2)
monkeypatch.setattr(
engram_ops,
"get_engram_dp_group",
lambda: SimpleNamespace(rank_in_group=0, broadcast_object=broadcast),
)
if query_fails:
monkeypatch.setattr(engram_ops, "check_shm_free_space", query)
layout = SimpleNamespace(num_embeddings=(1 << 50,), head_dim=DIM)
assert not engram_ops.can_share_engram_tables(layout)
assert len(broadcasts) == 1 and broadcasts[0] is not None
@pytest.mark.parametrize(
+15 -9
View File
@@ -8,6 +8,7 @@ Run: pytest tests/distributed/test_mnnvl_alltoall.py -v
import os
import traceback
from contextlib import contextmanager
import pytest
import torch
@@ -148,16 +149,17 @@ def _init_dp_environment(world_size, rank, port, dp_size, dp_port):
ensure_model_parallel_initialized(1, 1)
@contextmanager
def _make_forward_context(rank, world_size, num_tokens_per_rank):
"""Create a forward context with mock DP metadata for AgRs tests.
"""Keep the vLLM config and mock-DP forward context active together.
Returns a context manager suitable for ``with`` statements.
The real DPMetadata (with sp_local_sizes etc.) is created internally
by set_forward_context from num_tokens_across_dp; the attn_metadata
placeholder just satisfies the "attn_metadata is not None" guard.
"""
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.config.parallel import ParallelConfig
from vllm.config.vllm import VllmConfig
from vllm.forward_context import set_forward_context
class _AttnMeta:
@@ -173,14 +175,18 @@ def _make_forward_context(rank, world_size, num_tokens_per_rank):
is_moe_model=True,
data_parallel_rank=rank,
)
return set_forward_context(
_AttnMeta(),
vllm_config,
num_tokens=num_tokens_per_rank,
num_tokens_across_dp=torch.tensor(
[num_tokens_per_rank] * world_size, dtype=torch.int
with (
set_current_vllm_config(vllm_config),
set_forward_context(
_AttnMeta(),
vllm_config,
num_tokens=num_tokens_per_rank,
num_tokens_across_dp=torch.tensor(
[num_tokens_per_rank] * world_size, dtype=torch.int
),
),
)
):
yield
# ---------------------------------------------------------------------------
+3 -3
View File
@@ -108,10 +108,10 @@ class TestSingleWriterShmRingBuffer(unittest.TestCase):
# Clear the buffer
self.ring_buffer.clear()
# Check that metadata is empty and IDs reset
# Check that metadata is empty and IDs keep increasing
self.assertEqual(len(self.ring_buffer.metadata), 0)
self.assertEqual(self.ring_buffer.monotonic_id_start, 0)
self.assertEqual(self.ring_buffer.monotonic_id_end, 0)
self.assertEqual(self.ring_buffer.monotonic_id_start, 3)
self.assertEqual(self.ring_buffer.monotonic_id_end, 3)
self.assertEqual(self.ring_buffer.data_buffer_start, 0)
self.assertEqual(self.ring_buffer.data_buffer_end, 0)
+64 -5
View File
@@ -227,10 +227,68 @@ class TestSingleWriterShmObjectStorage(unittest.TestCase):
self.assertEqual(len(self.storage.id_index), 0)
self.assertEqual(len(self.storage.ring_buffer.metadata), 0)
# Verify that new items can be added after clearing
# Verify that new items can be added after clearing. IDs keep
# increasing so handles issued before the clear cannot alias new data.
address, monotonic_id = self.storage.put("new_item", "new_value")
self.assertIn("new_item", self.storage.key_index)
self.assertEqual((address, monotonic_id), (0, 0))
self.assertEqual((address, monotonic_id), (0, 5))
def _open_reader(self) -> SingleWriterShmObjectStorage:
reader = SingleWriterShmObjectStorage.create_from_handle(self.storage.handle())
self.addCleanup(reader.close)
return reader
def test_reader_get_and_touch_each_require_valid_signature(self):
address, monotonic_id = self.storage.put("key", "value")
signature = self.storage.get_signature("key")
reader = self._open_reader()
reader.touch("key", address, monotonic_id, signature)
self.assertEqual(reader.get(address, monotonic_id, signature, "key"), "value")
flipped = [signature[0] ^ 1, *signature[1:]]
for bad in (None, flipped, [signature], signature[:-1], [256] * 32):
with self.assertRaisesRegex(ValueError, "SHM handle signature"):
reader.get(address, monotonic_id, bad, "key")
with self.assertRaisesRegex(ValueError, "SHM handle signature"):
reader.touch("key", address, monotonic_id, bad)
def test_signature_is_bound_to_key_and_address(self):
address, monotonic_id = self.storage.put("key", "value")
signature = self.storage.get_signature("key")
reader = self._open_reader()
with self.assertRaisesRegex(ValueError, "SHM handle signature"):
reader.get(address, monotonic_id, signature, "other_key")
with self.assertRaisesRegex(ValueError, "SHM handle signature"):
reader.get(address + 1, monotonic_id, signature, "key")
def test_writer_rejects_handle_no_longer_in_key_index(self):
address, monotonic_id = self.storage.put("key", "value")
signature = self.storage.get_signature("key")
self.storage.verify_signature("key", address, monotonic_id, signature)
self.storage.clear()
with self.assertRaisesRegex(ValueError, "SHM handle signature"):
self.storage.verify_signature("key", address, monotonic_id, signature)
def test_handle_issued_before_clear_reads_own_data_until_reused(self):
address, monotonic_id = self.storage.put("key", "value")
signature = self.storage.get_signature("key")
reader = self._open_reader()
self.storage.clear()
# Requests still draining after a clear keep reading their own data.
self.assertEqual(reader.get(address, monotonic_id, signature, "key"), "value")
self.storage.put("key", "new_value")
with self.assertRaisesRegex(ValueError, "modified or is invalid"):
reader.get(address, monotonic_id, signature, "key")
with self.assertRaisesRegex(ValueError, "modified or is invalid"):
reader.touch("key", address, monotonic_id, signature)
# Reader process function
@@ -242,11 +300,11 @@ def reader_process(process_id, storage_handle, items_to_read):
errors = []
for key, original_value, address, monotonic_id in items_to_read:
for key, original_value, address, monotonic_id, signature in items_to_read:
time.sleep(random.random() / 100)
try:
# Read data from shared memory
retrieved_value = reader_storage.get(address, monotonic_id)
retrieved_value = reader_storage.get(address, monotonic_id, signature, key)
# Verify data integrity
assert retrieved_value == original_value
@@ -289,7 +347,8 @@ def run_multiprocess_example():
for key, value in test_data:
print(f"Storing {key}: {value}")
address, monotonic_id = storage.put(key, value)
stored_items.append((key, value, address, monotonic_id))
signature = storage.get_signature(key)
stored_items.append((key, value, address, monotonic_id, signature))
print(f" -> Stored at address {address}, ID {monotonic_id}")
print("\n--- Retrieving Data ---")
+396 -5
View File
@@ -6,10 +6,14 @@ Unit tests for engine classes (parsing, validation, registry).
Integration tests for NCCL and IPC weight transfer between processes using Ray.
"""
import builtins
import pickle
import runpy
import threading
import time
from contextlib import nullcontext
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import MagicMock
import pybase64 as base64
@@ -64,6 +68,7 @@ from vllm.distributed.weight_transfer.sparse_nccl_engine import (
SparseWeightPatch,
)
from vllm.platforms import current_platform
from vllm.utils.nccl import _nccl_has_no_cache
from vllm.utils.network_utils import get_open_port
@@ -297,6 +302,36 @@ class TestEngineRegistry:
)
assert isinstance(engine, SparseNCCLWeightTransferEngine)
def test_modelexpress_registration_is_native_and_lazy(self, monkeypatch):
from vllm.distributed.weight_transfer import factory
imported = []
module = MagicMock()
module.ModelExpressWeightTransferEngine.__name__ = (
"ModelExpressWeightTransferEngine"
)
def import_module(name):
imported.append(name)
return module
monkeypatch.setattr(factory.importlib, "import_module", import_module)
config = WeightTransferConfig(backend="modelexpress")
vllm_config = create_mock_vllm_config()
device = torch.device("cpu")
model = MagicMock(spec=torch.nn.Module)
assert imported == []
engine = WeightTransferEngineFactory.create_engine(
config, vllm_config, device, model
)
assert imported == ["vllm.distributed.weight_transfer.modelexpress_engine"]
module.ModelExpressWeightTransferEngine.assert_called_once_with(
config, vllm_config, device, model
)
assert engine is module.ModelExpressWeightTransferEngine.return_value
def test_create_engine_invalid_backend(self):
config = WeightTransferConfig(backend="invalid")
with pytest.raises(ValueError, match="Invalid weight transfer backend"):
@@ -399,13 +434,15 @@ def trainer_broadcast_tensor(
return True
@ray.remote(num_gpus=1)
# max_calls=1: a batch-invariant run leaves NCCL pins in the worker process.
@ray.remote(num_gpus=1, max_calls=1)
def inference_receive_tensor(
master_address: str,
master_port: int,
world_size: int,
tensor_shape: list[int],
tensor_dtype: str,
batch_invariant: bool = False,
) -> dict:
"""Inference task that receives tensor via NCCLWeightTransferEngine."""
import contextlib
@@ -414,6 +451,21 @@ def inference_receive_tensor(
import torch
_set_ray_assigned_device()
if batch_invariant:
from vllm.model_executor.determinism.batch_invariant import (
override_envs_for_invariance,
)
override_envs_for_invariance()
# Like vLLM's own groups, a pinned communicator makes NCCL read the
# pins before the transfer group exists.
torch.distributed.init_process_group(
"nccl",
init_method=f"tcp://127.0.0.1:{get_open_port()}",
rank=0,
world_size=1,
)
torch.distributed.all_reduce(torch.ones(1, device="cuda"))
from vllm.config.parallel import ParallelConfig
from vllm.config.weight_transfer import WeightTransferConfig
@@ -500,11 +552,25 @@ def inference_receive_tensor(
torch.accelerator.device_count() < 2,
reason="Need at least 2 GPUs to run NCCL weight transfer test.",
)
def test_nccl_weight_transfer_between_processes():
@pytest.mark.parametrize(
"batch_invariant",
[
pytest.param(False, id="default"),
pytest.param(
True,
id="batch-invariant-worker",
marks=pytest.mark.skipif(
not _nccl_has_no_cache(), reason="Needs CUDA NCCL >= 2.29.7."
),
),
],
)
def test_nccl_weight_transfer_between_processes(batch_invariant):
"""Test NCCL weight transfer from trainer to inference process using Ray.
This test verifies that the NCCLWeightTransferEngine can receive
tensors broadcast by a trainer process via NCCL.
tensors broadcast by a trainer process via NCCL, including when the
worker runs with batch-invariance NCCL pins the trainer does not have.
"""
_init_ray_for_weight_transfer()
@@ -516,13 +582,19 @@ def test_nccl_weight_transfer_between_processes():
tensor_dtype = "float32"
inference_future = inference_receive_tensor.remote(
master_address, master_port, world_size, tensor_shape, tensor_dtype
master_address,
master_port,
world_size,
tensor_shape,
tensor_dtype,
batch_invariant,
)
trainer_future = trainer_broadcast_tensor.remote(
master_address, master_port, world_size, tensor_shape, tensor_dtype
)
trainer_result, result = ray.get([trainer_future, inference_future])
# A mismatched NCCL config deadlocks instead of failing.
trainer_result, result = ray.get([trainer_future, inference_future], timeout=300)
assert trainer_result, "Trainer should complete successfully"
assert result["success"], (
@@ -1976,3 +2048,322 @@ def test_sparse_nccl_trainer_non_sender_skips_client():
engine.send_weights([_sparse_patch()])
engine.send_weight_chunk([_sparse_patch()])
assert client.order == []
# --- Optional ModelExpress Client Lifecycle ---
def _import_modelexpress_shim():
path = (
Path(__file__).parents[2]
/ "vllm/distributed/weight_transfer/modelexpress_engine.py"
)
runpy.run_path(str(path), run_name="_test_modelexpress_shim")
@pytest.mark.parametrize(
"missing_module",
[
"modelexpress",
"modelexpress_rl",
"modelexpress_rl.inference",
"modelexpress_rl.inference.engines",
"modelexpress_rl.inference.engines.vllm",
"modelexpress_rl.inference.engines.vllm.weight_transfer_engine",
],
)
def test_modelexpress_missing_backend_explains_source_install(
monkeypatch, missing_module
):
original_import = builtins.__import__
error = ModuleNotFoundError(name=missing_module)
def import_without_backend(name, *args, **kwargs):
if name == "modelexpress":
raise error
return original_import(name, *args, **kwargs)
monkeypatch.setattr(builtins, "__import__", import_without_backend)
with pytest.raises(ImportError, match="modelexpress_client/python") as exc_info:
_import_modelexpress_shim()
assert exc_info.value.__cause__ is error
assert "uv pip install" in str(exc_info.value)
@pytest.mark.parametrize("missing_module", ["grpc", "modelexpress.internal_dependency"])
def test_modelexpress_preserves_transitive_import_errors(monkeypatch, missing_module):
original_import = builtins.__import__
error = ModuleNotFoundError(name=missing_module)
def import_without_dependency(name, *args, **kwargs):
if name == "modelexpress":
raise error
return original_import(name, *args, **kwargs)
monkeypatch.setattr(builtins, "__import__", import_without_dependency)
with pytest.raises(ModuleNotFoundError) as exc_info:
_import_modelexpress_shim()
assert exc_info.value is error
@pytest.fixture
def mx_client(monkeypatch):
client_module = pytest.importorskip("modelexpress_rl.inference.client")
client = MagicMock()
monkeypatch.setattr(
client_module.ModelExpressGeneratorClient,
"initialize",
MagicMock(return_value=client),
)
monkeypatch.setattr(torch.accelerator, "synchronize", MagicMock())
return client
def _make_modelexpress_engine():
return WeightTransferEngineFactory.create_engine(
WeightTransferConfig(backend="modelexpress"),
SimpleNamespace(
parallel_config=SimpleNamespace(),
model_config=SimpleNamespace(model="test/model"),
),
torch.device("cpu"),
torch.nn.Linear(2, 2),
)
@pytest.fixture
def mx_worker(mx_client, monkeypatch):
from vllm.v1.worker import gpu_worker
monkeypatch.setattr(gpu_worker, "set_current_vllm_config", lambda _: nullcontext())
worker = object.__new__(gpu_worker.Worker)
worker.weight_transfer_engine = _make_modelexpress_engine()
worker.vllm_config = worker.weight_transfer_engine.vllm_config
worker._weight_update_active = False
worker._weight_update_is_draft = False
worker.model_runner = MagicMock()
return worker
def test_modelexpress_worker_rejects_updates_before_initialization(
mx_worker, mx_client
):
with pytest.raises(RuntimeError, match="not initialized"):
mx_worker.start_weight_update()
with pytest.raises(RuntimeError, match="start_weight_update must be called"):
mx_worker.update_weights({"version_id": "version-a"})
with pytest.raises(RuntimeError, match="without a matching"):
mx_worker.finish_weight_update()
mx_client.stage_weight.assert_not_called()
@pytest.mark.parametrize(
"second_update, error_type",
[
({"version_id": "version-a"}, RuntimeError),
({"version_id": "version-b"}, RuntimeError),
({"version_id": ""}, ValueError),
({"version_id": None}, ValueError),
({}, ValueError),
],
)
def test_modelexpress_worker_recovers_after_rejected_update(
mx_worker, mx_client, second_update, error_type
):
"""A rejected payload must release A so a fresh session can actually install B."""
mx_worker.weight_transfer_engine.init_transfer_engine(
mx_worker.weight_transfer_engine.parse_init_info({})
)
staged_a = SimpleNamespace(version_id="version-a", metrics={}, release=MagicMock())
staged_b = SimpleNamespace(version_id="version-b", metrics={}, release=MagicMock())
mx_client.stage_weight.side_effect = [staged_a, staged_b]
mx_client.apply_weight.return_value = {}
mx_worker.start_weight_update()
mx_worker.update_weights({"version_id": "version-a"})
with pytest.raises(error_type):
mx_worker.update_weights(second_update)
staged_a.release.assert_called_once_with()
mx_client.apply_weight.assert_called_once_with(staged_a)
mx_worker.start_weight_update()
mx_worker.update_weights({"version_id": "version-b"})
mx_worker.finish_weight_update()
assert [
call.kwargs["version"].version_id
for call in mx_client.stage_weight.call_args_list
] == ["version-a", "version-b"]
mx_client.apply_weight.assert_called_with(staged_b)
staged_b.release.assert_called_once_with()
def test_modelexpress_worker_can_retry_failed_finish(mx_worker, mx_client):
mx_worker.weight_transfer_engine.init_transfer_engine(
mx_worker.weight_transfer_engine.parse_init_info({})
)
staged = mx_client.stage_weight.return_value
staged.release.side_effect = [RuntimeError("release failed"), None]
mx_worker.start_weight_update()
mx_worker.update_weights({"version_id": "version-a"})
with pytest.raises(RuntimeError, match="release failed"):
mx_worker.finish_weight_update()
mx_worker.finish_weight_update()
assert staged.release.call_count == 2
assert not mx_worker._weight_update_active
mx_worker.start_weight_update()
def test_native_engine_applies_and_releases_exact_versions(mx_client):
from modelexpress_rl.inference.engines.vllm import (
weight_transfer_engine as mx_engine,
)
from vllm.distributed.weight_transfer import modelexpress_engine
engine = _make_modelexpress_engine()
assert type(engine) is mx_engine.ModelExpressWeightTransferEngine
assert (
modelexpress_engine.ModelExpressWeightTransferInitInfo
is mx_engine.ModelExpressWeightTransferInitInfo
)
assert (
modelexpress_engine.ModelExpressWeightTransferUpdateInfo
is mx_engine.ModelExpressWeightTransferUpdateInfo
)
assert not engine.supports_draft_weight_update
engine.init_transfer_engine(engine.parse_init_info({}))
for version_id in ("version-a", "version-b"):
staged = SimpleNamespace(version_id=version_id, metrics={}, release=MagicMock())
mx_client.stage_weight.return_value = staged
mx_client.apply_weight.return_value = {}
engine.start_weight_update()
engine.update_weights({"version_id": version_id})
assert (
mx_client.stage_weight.call_args.kwargs["version"].version_id == version_id
)
mx_client.apply_weight.assert_called_with(staged)
torch.accelerator.synchronize.assert_called()
staged.release.assert_not_called()
engine.finish_weight_update()
staged.release.assert_called_once_with()
engine.shutdown()
engine.shutdown()
mx_client.close.assert_called_once_with()
def test_vime_init_config_reaches_mx_client(mx_client):
from modelexpress_rl.inference.client import ModelExpressGeneratorClient
engine = _make_modelexpress_engine()
ModelExpressGeneratorClient.initialize.assert_not_called()
engine.init_transfer_engine(
engine.parse_init_info(
{
"model_name": "policy",
"server_url": "mx:8001",
"initial_serving_version_id": "serving-a",
"object_storage_type": "S3",
"initial_base_version_id": "base-a",
"seed_checkpoint_path": "/models/launch",
"refit_checkpoint_dir": "/mxdelta/receiver",
"refit_checkpoint_max_size_gb": 200,
"object_storage_endpoint_url": "http://minio:9000",
"object_storage_region_name": "us-west-2",
"registration_ttl_seconds": 90,
"lease_ttl_seconds": 60,
"max_transfer_attempts": 4,
"max_replay_chain_length": 17,
"rpc_timeout_seconds": 12.5,
}
)
)
config = ModelExpressGeneratorClient.initialize.call_args.args[0]
assert config.engine_context.model is engine.model
assert config.engine_context.vllm_config is engine.vllm_config
assert config.model_name == "policy"
assert config.server_url == "mx:8001"
assert config.initial_serving_version_id == "serving-a"
assert config.registration_ttl_seconds == 90
assert config.lease_ttl_seconds == 60
assert config.max_transfer_attempts == 4
assert config.max_replay_chain_length == 17
assert config.rpc_timeout_seconds == 12.5
storage = config.object_storage
assert storage.storage_type.value == "S3"
assert storage.initial_base_version_id == "base-a"
assert storage.seed_checkpoint_path == "/models/launch"
assert storage.refit_checkpoint_dir == "/mxdelta/receiver"
assert storage.refit_checkpoint_max_size_gb == 200
assert storage.endpoint_url == "http://minio:9000"
assert storage.region_name == "us-west-2"
@pytest.mark.parametrize(
"key",
[
"object_storage_type",
"initial_base_version_id",
"seed_checkpoint_path",
"refit_checkpoint_dir",
"object_storage_endpoint_url",
"object_storage_region_name",
],
)
def test_incomplete_storage_config_rejected_before_init(mx_client, key):
from modelexpress_rl.inference.client import ModelExpressGeneratorClient
engine = _make_modelexpress_engine()
with pytest.raises(ValueError, match="object storage requires"):
engine.init_transfer_engine(engine.parse_init_info({key: "value"}))
ModelExpressGeneratorClient.initialize.assert_not_called()
@pytest.mark.parametrize("version_id", ["", " ", None, 1])
def test_invalid_version_rejected_before_staging(mx_client, version_id):
engine = _make_modelexpress_engine()
engine.init_transfer_engine(engine.parse_init_info({}))
engine.start_weight_update()
with pytest.raises(ValueError, match="version_id is required"):
engine.update_weights({"version_id": version_id})
mx_client.stage_weight.assert_not_called()
@pytest.mark.parametrize("failure", ["stage", "apply", "release"])
def test_update_failure_preserves_error_and_releases_handle(mx_client, failure):
engine = _make_modelexpress_engine()
engine.init_transfer_engine(engine.parse_init_info({}))
staged = SimpleNamespace(version_id="version-a", metrics={}, release=MagicMock())
mx_client.stage_weight.return_value = staged
if failure == "stage":
mx_client.stage_weight.side_effect = RuntimeError("stage failed")
else:
mx_client.apply_weight.side_effect = RuntimeError("apply failed")
if failure == "release":
staged.release.side_effect = RuntimeError("release failed")
engine.start_weight_update()
with pytest.raises(RuntimeError, match="stage failed|apply failed"):
engine.update_weights({"version_id": "version-a"})
assert staged.release.call_count == (0 if failure == "stage" else 1)
torch.accelerator.synchronize.assert_not_called()
engine.shutdown()
mx_client.close.assert_called_once_with()
def test_plugin_does_not_replace_native_backend(mx_client):
from modelexpress.engines.vllm.registration import (
register_plugin_weight_transfer_engine,
)
from modelexpress_rl.inference.engines.vllm.weight_transfer_engine import (
ModelExpressWeightTransferEngine,
)
native_loader = WeightTransferEngineFactory._registry["modelexpress"]
register_plugin_weight_transfer_engine()
assert WeightTransferEngineFactory._registry["modelexpress"] is native_loader
assert type(_make_modelexpress_engine()) is ModelExpressWeightTransferEngine
@@ -27,6 +27,7 @@ from pydantic import BaseModel, Field, ValidationError
from vllm.entrypoints.anthropic.api_router import attach_router
from vllm.entrypoints.anthropic.protocol import (
AnthropicCountTokensRequest,
AnthropicMessagesRequest,
)
from vllm.entrypoints.anthropic.serving import (
@@ -37,6 +38,8 @@ from vllm.entrypoints.generate.base.protocol import (
DeltaFunctionCall,
DeltaMessage,
DeltaToolCall,
FunctionCall,
ToolCall,
)
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionResponse,
@@ -353,9 +356,11 @@ class TestToolResultContent:
if m["role"] == "user" and isinstance(m.get("content"), list)
]
assert len(follow_up) == 1
assert follow_up[0]["content"][0]["image_url"]["url"] == (
"data:image/jpeg;base64,QUFB"
)
content = follow_up[0]["content"]
assert isinstance(content, list)
image = content[0]
assert image["type"] == "image_url"
assert image["image_url"]["url"] == "data:image/jpeg;base64,QUFB"
def test_tool_result_with_multiple_images(self):
request = self._make_tool_result_request(
@@ -385,7 +390,12 @@ class TestToolResultContent:
if m["role"] == "user" and isinstance(m.get("content"), list)
]
assert len(follow_up) == 1
urls = [p["image_url"]["url"] for p in follow_up[0]["content"]]
content = follow_up[0]["content"]
assert isinstance(content, list)
urls = []
for part in content:
assert part["type"] == "image_url"
urls.append(part["image_url"]["url"])
assert urls == [
"data:image/png;base64,IMG1",
"https://example.com/img2.jpg",
@@ -582,7 +592,8 @@ class TestThinkingBlockConversion:
asst = asst_msgs[0]
assert asst.get("reasoning") == "I need to call the calculator."
tool_calls = list(asst.get("tool_calls", []))
tool_calls = asst.get("tool_calls")
assert tool_calls is not None
assert len(tool_calls) == 1
assert tool_calls[0]["function"]["name"] == "calculator"
# No text content alongside reasoning + tool_use.
@@ -999,11 +1010,6 @@ class TestInlineSystemMessageInMessagesArray:
def _make_stream_converter():
obj = MagicMock(spec=AnthropicServingMessages)
obj.stop_reason_map = {
"stop": "end_turn",
"length": "max_tokens",
"tool_calls": "tool_use",
}
obj.message_stream_converter = (
AnthropicServingMessages.message_stream_converter.__get__(obj)
)
@@ -1445,7 +1451,7 @@ class TestDetectMergeInlineSystem:
AnthropicServingMessages._should_merge_inline_system(online_renderer)
is expected
)
assert mock_logger.warning.called is expected
assert mock_logger.warning_once.called is expected
# ======================================================================
@@ -1671,6 +1677,78 @@ class TestStopSequenceReason:
assert msg_deltas[0]["delta"]["stop_sequence"] is None
class TestToolUseStopReason:
"""A response carrying a tool_use block reports ``stop_reason="tool_use"``
even when the chat layer says ``finish_reason="stop"``, as it does for a
forced (named) tool. A call cut short by max_tokens or a stop string does not.
"""
@pytest.mark.parametrize(
("finish_reason", "stop_reason", "expected"),
[
("stop", None, "tool_use"),
("tool_calls", "</x>", "tool_use"),
],
)
def test_non_streaming(self, finish_reason, stop_reason, expected):
tool_call = ToolCall(
function=FunctionCall(name="get_weather", arguments='{"city": "Paris"}')
)
response = ChatCompletionResponse(
id="chatcmpl-test",
model="test-model",
choices=[
ChatCompletionResponseChoice(
index=0,
message=ChatMessage(role="assistant", tool_calls=[tool_call]),
finish_reason=finish_reason,
stop_reason=stop_reason,
)
],
usage=UsageInfo(prompt_tokens=5, total_tokens=8, completion_tokens=3),
)
result = _make_full_converter().messages_full_converter(response)
assert result.content[0].type == "tool_use"
assert result.stop_reason == expected
@pytest.mark.asyncio
@pytest.mark.parametrize(
("args", "finish_reason", "stop_reason", "expected"),
[
('{"city": "Paris"}', "stop", None, "tool_use"),
('{"city": "Pa', "length", None, "max_tokens"),
('{"city": "Paris"}', "tool_calls", "</x>", "tool_use"),
('{"city": "Pa', "tool_calls", "</x>", "stop_sequence"),
],
)
async def test_streaming(self, args, finish_reason, stop_reason, expected):
async def sse_input():
yield _make_stream_chunk(delta=DeltaMessage(role="assistant"))
yield _make_stream_chunk(
delta=DeltaMessage(
tool_calls=[_tc(id="call_1", name="get_weather", args=args)]
)
)
yield _make_stream_chunk(
finish_reason=finish_reason, stop_reason=stop_reason
)
yield _make_stream_chunk(
choices=[],
usage=UsageInfo(prompt_tokens=5, total_tokens=8, completion_tokens=3),
)
yield "data: [DONE]"
converter = _make_stream_converter()
events = _parse_sse_events(
[e async for e in converter.message_stream_converter(sse_input())]
)
msg_deltas = [data for ev_type, data in events if ev_type == "message_delta"]
assert msg_deltas[0]["delta"]["stop_reason"] == expected
# ======================================================================
# Client-caused errors are 4xx, not 500 (Issue #52088)
# ======================================================================
@@ -1955,3 +2033,87 @@ class TestProbeDisabledThinkingEffort:
@pytest.mark.asyncio
async def test_renderer_rejects_none(self):
assert await self._probe(self._reject_none) == "low"
class TestMidConversationToolChanges:
"""``tool_addition``/``tool_removal`` blocks decide which tools the chat
template is given, since it has no other way to see them."""
@staticmethod
def _convert_tools(*changes: dict) -> dict[str, bool | None]:
request = _make_request(
[
{"role": "user", "content": "Hello"},
{
"role": "system",
"content": [{"type": "text", "text": "Tools changed."}, *changes],
},
],
tools=[
{"name": "bash", "input_schema": {}},
{"name": "search", "input_schema": {}, "defer_loading": True},
{"name": "fetch", "input_schema": {}, "defer_loading": True},
],
)
result = _convert(request)
assert result.messages[-1] == {"role": "system", "content": "Tools changed."}
assert result.tools is not None
return {t.function.name: t.function.defer_loading for t in result.tools}
@staticmethod
def _change(block_type: str, name: str) -> dict:
return {"type": block_type, "tool": {"type": "tool_reference", "name": name}}
def test_addition_loads_deferred_tool(self):
tools = self._convert_tools(self._change("tool_addition", "search"))
assert tools == {"bash": None, "search": None, "fetch": True}
def test_removal_withdraws_tool_until_added_again(self):
removal = self._change("tool_removal", "bash")
assert "bash" not in self._convert_tools(removal)
readded = self._convert_tools(removal, self._change("tool_addition", "bash"))
assert list(readded) == ["bash", "search", "fetch"]
def test_addition_defines_tool_by_value(self):
definition = {"name": "db_query", "input_schema": {}}
tools = self._convert_tools(
{
"type": "tool_addition",
"tool": {"type": "tool_definition", "definition": definition},
}
)
assert list(tools) == ["bash", "search", "fetch", "db_query"]
def test_unknown_tool_reference_is_rejected(self):
with pytest.raises(ValueError, match="not declared in tools"):
self._convert_tools(self._change("tool_addition", "missing"))
def test_removal_by_value_is_rejected(self):
definition = {"name": "bash", "input_schema": {}}
with pytest.raises(ValidationError, match="only accepts a tool_reference"):
self._convert_tools(
{
"type": "tool_removal",
"tool": {"type": "tool_definition", "definition": definition},
}
)
@pytest.mark.parametrize(
"request_cls", [AnthropicMessagesRequest, AnthropicCountTokensRequest]
)
@pytest.mark.parametrize("in_top_level_system", [False, True])
def test_tool_change_outside_system_message_is_rejected(
self, request_cls, in_top_level_system
):
change = self._change("tool_addition", "search")
if in_top_level_system:
fields = {
"messages": [{"role": "user", "content": "Hi"}],
"system": [change],
}
else:
fields = {"messages": [{"role": "user", "content": [change]}]}
with pytest.raises(
ValidationError, match='only allowed in messages with role "system"'
):
request_cls(model="test-model", max_tokens=128, **fields)
+1 -1
View File
@@ -224,7 +224,7 @@ async def test_anthropic_streaming_cache_usage(client: anthropic.AsyncAnthropic)
request = dict(
model="claude-3-7-sonnet-latest",
max_tokens=1,
temperature=0.0,
extra_body={"temperature": 0.0},
messages=[
{
"role": "user",
@@ -29,7 +29,7 @@ from vllm.entrypoints.cohere.cohere_chat_message import (
class TestCitationSource:
def test_invalid_type_rejected(self):
with pytest.raises(ValidationError):
CitationSource(type="other") # type: ignore[arg-type]
CitationSource(type="other")
def test_none_fields_excluded_from_dump(self):
s = CitationSource(type="document", id="d1")
@@ -65,7 +65,7 @@ class TestCitationSource:
class TestCitation:
def test_invalid_type_rejected(self):
with pytest.raises(ValidationError):
Citation(type="OTHER") # type: ignore[arg-type]
Citation(type="OTHER")
def test_dump_excludes_none_fields(self):
c = Citation(start=0, end=5, text="hello")
+2 -2
View File
@@ -136,7 +136,7 @@ class TestCohereChatV2Response:
with pytest.raises(ValidationError):
CohereChatV2Response(
id="r1",
finish_reason="NOT_A_REASON", # type: ignore[arg-type]
finish_reason="NOT_A_REASON",
message=msg,
)
@@ -246,5 +246,5 @@ class TestStreamingEventTypeField:
MessageStartEvent(
id="a",
delta={"message": {"role": "assistant"}},
type="other", # type: ignore[arg-type]
type="other",
)
-120
View File
@@ -14,16 +14,6 @@ def sample_prompts():
]
@pytest.fixture
def sample_token_ids():
return [
[0],
[0, 1],
[0, 2, 1],
[0, 3, 1, 2],
]
@pytest.fixture
def sample_regex():
return (
@@ -61,104 +51,6 @@ def sample_json_schema():
}
@pytest.fixture
def sample_complex_json_schema():
return {
"type": "object",
"properties": {
"score": {
"type": "integer",
"minimum": 0,
"maximum": 100, # Numeric range
},
"grade": {
"type": "string",
"pattern": "^[A-D]$", # Regex pattern
},
"email": {
"type": "string",
"pattern": "^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}$",
},
"tags": {
"type": "array",
"items": {
"type": "string",
# Combining length and pattern restrictions
"pattern": "^[a-z]{1,10}$",
},
},
},
"required": ["score", "grade", "email", "tags"],
}
@pytest.fixture
def sample_definition_json_schema():
return {
"$defs": {
"Step": {
"properties": {
"explanation": {"title": "Explanation", "type": "string"},
"output": {"title": "Output", "type": "string"},
},
"required": ["explanation", "output"],
"title": "Step",
"type": "object",
}
},
"properties": {
"steps": {
"items": {"$ref": "#/$defs/Step"},
"title": "Steps",
"type": "array",
},
"final_answer": {"title": "Final Answer", "type": "string"},
},
"required": ["steps", "final_answer"],
"title": "MathReasoning",
"type": "object",
}
@pytest.fixture
def sample_enum_json_schema():
return {
"type": "object",
"properties": {
"status": {
"type": "string",
"enum": ["active", "inactive", "pending"], # Literal values using enum
},
"priority": {
"type": "string",
"enum": ["low", "medium", "high", "critical"],
},
"category": {
"type": "object",
"properties": {
"type": {
"type": "string",
"enum": ["bug", "feature", "improvement"],
},
"severity": {
"type": "integer",
"enum": [1, 2, 3, 4, 5], # Enum can also contain numbers
},
},
"required": ["type", "severity"],
},
"flags": {
"type": "array",
"items": {
"type": "string",
"enum": ["urgent", "blocked", "needs_review", "approved"],
},
},
},
"required": ["status", "priority", "category", "flags"],
}
@pytest.fixture
def sample_structured_outputs_choices():
return [
@@ -175,18 +67,6 @@ def sample_structured_outputs_choices():
]
@pytest.fixture
def sample_sql_statements():
return """
start: select_statement
select_statement: "SELECT" column "from" table "where" condition
column: "col_1" | "col_2"
table: "table_1" | "table_2"
condition: column "=" number
number: "1" | "2"
"""
@pytest.fixture(scope="session")
def qwen3_lora_files():
"""Download Qwen3 LoRA files once per test session."""
@@ -307,7 +307,8 @@ def test_external_process_monitoring(api_server_args):
def run_with_exception_capture():
try:
wait_for_completion_or_failure(
api_server_manager=manager, coordinator=mock_coordinator
api_server_manager=manager,
coordinator=mock_coordinator,
)
except Exception as e:
result["exception"] = e
@@ -489,3 +490,102 @@ def test_rust_frontend_launch_log_redacts_credentials(monkeypatch, caplog):
assert hf_token not in message
assert api_key not in message
assert '"hf_token": "***"' in message
def test_rust_frontend_uses_config_model_as_model_tag(monkeypatch, caplog, tmp_path):
"""Config-only model selection must be forwarded to the Rust frontend."""
import subprocess as subprocess_mod
from vllm.entrypoints.launchers.cli_args import make_arg_parser
from vllm.utils.argparse_utils import FlexibleArgumentParser
from vllm.v1.utils import RustFrontendProcessManager
config_path = tmp_path / "config.yml"
config_path.write_text("model: org/model\n")
args = make_arg_parser(FlexibleArgumentParser()).parse_args(
["--config", str(config_path)]
)
assert args.model == "org/model"
assert args.model_tag is None
class _FakeProc:
pid = 4321
returncode = 0
def poll(self):
return 0
def wait(self, timeout=None):
return 0
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
monkeypatch.setattr(subprocess_mod, "Popen", lambda *a, **kw: _FakeProc())
with caplog.at_level("INFO", logger="vllm.v1.utils"):
RustFrontendProcessManager(
binary_path="/nonexistent/vllm-rs",
sock=sock,
args=args,
input_address="ipc:///tmp/in",
output_address="ipc:///tmp/out",
engine_start_index=0,
engine_count=1,
data_parallel_size=1,
)
finally:
sock.close()
assert '"model_tag": "org/model"' in caplog.text
def test_rust_frontend_inherits_grpc_listener(monkeypatch):
"""The Python-bound gRPC socket is passed to Rust like the HTTP one."""
import subprocess as subprocess_mod
from vllm.entrypoints.launchers.cli_args import make_arg_parser
from vllm.utils.argparse_utils import FlexibleArgumentParser
from vllm.v1.utils import RustFrontendProcessManager
args = make_arg_parser(FlexibleArgumentParser()).parse_args(
["--model", "org/model", "--grpc-port", "50051"]
)
class _FakeProc:
pid = 4321
returncode = 0
def poll(self):
return 0
def wait(self, timeout=None):
return 0
popen_calls = []
def fake_popen(cmd, **kwargs):
popen_calls.append((cmd, kwargs))
return _FakeProc()
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
grpc_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
monkeypatch.setattr(subprocess_mod, "Popen", fake_popen)
RustFrontendProcessManager(
binary_path="/nonexistent/vllm-rs",
sock=sock,
args=args,
input_address="ipc:///tmp/in",
output_address="ipc:///tmp/out",
engine_start_index=0,
engine_count=1,
data_parallel_size=1,
grpc_sock=grpc_sock,
)
[(cmd, kwargs)] = popen_calls
grpc_fd = grpc_sock.fileno()
flag_index = cmd.index("--grpc-listen-fd")
assert cmd[flag_index + 1] == str(grpc_fd)
assert list(kwargs["pass_fds"]) == [sock.fileno(), grpc_fd]
finally:
sock.close()
grpc_sock.close()
+86 -6
View File
@@ -28,7 +28,7 @@ assert CHATML_JINJA_PATH.exists()
def _build_vllm_parsers():
vllm_parser = FlexibleArgumentParser()
subparsers = vllm_parser.add_subparsers()
subparsers = vllm_parser.add_subparsers(dest="subparser")
serve_parser = subparsers.add_parser("serve")
make_arg_parser(serve_parser)
return {"vllm": vllm_parser, "vllm serve": serve_parser}
@@ -76,6 +76,8 @@ def test_logging_config_cli_args(serve_parser):
[
"--logging-config",
'{"log_level":"WARNING","pylogging_config_file":"/tmp/json.json"}',
"--logging-config.formatter",
"json",
"--log-level",
"DEBUG",
"--log-config-file",
@@ -85,6 +87,7 @@ def test_logging_config_cli_args(serve_parser):
config = AsyncEngineArgs.from_cli_args(args).create_logging_config()
assert config.log_level == "DEBUG"
assert config.formatter == "json"
assert config.pylogging_config_file == "/tmp/flat.json"
@@ -170,6 +173,75 @@ def test_multiple_valid_inputs(serve_parser):
### Tests for serve argument validation that run prior to loading
@pytest.mark.parametrize(
"port, extra_args",
[
(50051, []),
(0, ["--port", "0"]),
(50051, ["--port", "50051", "--uds", "/tmp/vllm.sock"]),
],
)
def test_rust_grpc_port_accepts_frontend_launch(
vllm_parser, monkeypatch, port, extra_args
):
monkeypatch.setenv("VLLM_USE_RUST_FRONTEND", "1")
args = vllm_parser.parse_args(
[
"serve",
"--grpc-port",
str(port),
"--data-parallel-size",
"8",
"--data-parallel-size-local",
"4",
"--data-parallel-start-rank",
"4",
"--data-parallel-hybrid-lb",
*extra_args,
]
)
validate_parsed_serve_args(args)
assert args.grpc_port == port
@pytest.mark.parametrize(
"rust_enabled, extra_args, port",
[
(False, [], 50051),
(False, ["--grpc"], 50051),
(True, ["--grpc"], 50051),
(True, ["--headless"], 50051),
(True, ["--api-server-count", "0"], 50051),
(True, ["--api-server-count", "-1"], 50051),
(True, ["--data-parallel-multi-port-external-lb"], 50051),
(True, ["--port", "50051"], 50051),
(True, [], -1),
(True, [], 65536),
],
)
def test_rust_grpc_port_rejects_incompatible_launch(
vllm_parser, monkeypatch, rust_enabled, extra_args, port
):
monkeypatch.setenv("VLLM_USE_RUST_FRONTEND", "1" if rust_enabled else "0")
args = vllm_parser.parse_args(["serve", "--grpc-port", str(port), *extra_args])
error = (
"--grpc and --grpc-port are mutually exclusive"
if "--grpc" in extra_args
else "--grpc-port"
)
with pytest.raises(ValueError, match=error):
validate_parsed_serve_args(args)
def test_rust_grpc_port_requires_serve_subcommand(monkeypatch):
"""Direct Python render/API entrypoints must reject a Rust-only listener."""
monkeypatch.setenv("VLLM_USE_RUST_FRONTEND", "1")
parser = make_arg_parser(FlexibleArgumentParser())
args = parser.parse_args(["--grpc-port", "50051"])
with pytest.raises(ValueError, match="--grpc-port requires"):
validate_parsed_serve_args(args)
def test_enable_auto_choice_passes_without_tool_call_parser(serve_parser):
"""Ensure validation fails if tool choice is enabled with no call parser."""
# If we enable-auto-tool-choice, explode with no tool-call-parser
@@ -262,13 +334,21 @@ def launch_render_parser():
return _build_launch_render_parser()
def test_launch_render_validates_serve_args(launch_render_parser):
@pytest.mark.parametrize(
"cli_args, error",
[
(["--enable-auto-tool-choice"], TypeError),
(["--grpc-port", "50051"], ValueError),
],
)
def test_launch_render_validates_serve_args(
launch_render_parser, monkeypatch, cli_args, error
):
"""`vllm launch render` reuses the serve parser, so it gets the serve checks"""
args = launch_render_parser.parse_args(
args=["launch", "render", "--enable-auto-tool-choice"]
)
monkeypatch.setenv("VLLM_USE_RUST_FRONTEND", "1")
args = launch_render_parser.parse_args(args=["launch", "render", *cli_args])
assert args.subparser == "launch"
with pytest.raises(TypeError):
with pytest.raises(error):
validate_parsed_serve_args(args)
@@ -47,7 +47,6 @@ _SUPERVISOR_PORT = 19256
_CHILD_PORT_BASE = 18000
_N_CHILDREN = 2
_PROBE_INTERVAL = 1.0
_POLL_INTERVAL = 1.0
# ---------------------------------------------------------------------------
+44 -4
View File
@@ -14,6 +14,7 @@ import httpx
import openai
import psutil
import pytest
from prometheus_client.parser import text_string_to_metric_families
from tests.utils import RemoteOpenAIServer
from vllm.utils.network_utils import get_open_port
@@ -127,6 +128,40 @@ async def _concurrent_request_loop(
t.cancel()
def _num_running_requests(metrics_text: str) -> float:
"""Sum ``vllm:num_requests_running`` across the server's label sets."""
total = 0.0
for family in text_string_to_metric_families(metrics_text):
if family.name == "vllm:num_requests_running":
for sample in family.samples:
if sample.name == "vllm:num_requests_running":
total += sample.value
return total
async def _wait_for_running_request(
metrics_url: str,
timeout: float = _INFLIGHT_REQUEST_START_TIMEOUT,
) -> float:
"""Wait until the server reports a running request, returning the delay.
``ShutdownState.inflight_requests`` is incremented before the HTTP call is
awaited, so it only shows that a client coroutine started. Draining can
only complete work the engine has already admitted, so synchronize on the
server's own scheduler gauge instead.
"""
start = time.monotonic()
deadline = start + timeout
async with httpx.AsyncClient() as metrics_client:
while time.monotonic() < deadline:
response = await metrics_client.get(metrics_url)
response.raise_for_status()
if _num_running_requests(response.text) > 0:
return time.monotonic() - start
await asyncio.sleep(_INFLIGHT_REQUEST_POLL_INTERVAL)
raise AssertionError(f"No request reached the running state within {timeout}s")
@pytest.mark.asyncio
async def test_shutdown_on_engine_failure(tmp_path: Path):
"""Verify that API returns connection error when server process is killed.
@@ -240,11 +275,15 @@ async def test_wait_timeout_completes_requests():
_concurrent_request_loop(client, state, sigterm_sent, concurrency=10)
)
await asyncio.sleep(0.5)
proc.send_signal(signal.SIGTERM)
sigterm_sent.set()
try:
# Drain can only complete requests the engine already admitted, so
# wait for the server to report one instead of guessing with a sleep.
admission_delay = await _wait_for_running_request(
remote_server.url_for("metrics")
)
proc.send_signal(signal.SIGTERM)
sigterm_sent.set()
await asyncio.wait_for(request_task, timeout=_SHUTDOWN_DETECTION_TIMEOUT)
except asyncio.TimeoutError:
pass
@@ -257,6 +296,7 @@ async def test_wait_timeout_completes_requests():
# wait timeout should complete in-flight requests
assert state.requests_after_sigterm > 0, (
f"Wait timeout should complete in-flight requests. "
f"admission_delay: {admission_delay:.3f}s, "
f"503: {state.got_503}, 500: {state.got_500}, "
f"conn_errors: {state.connection_errors}, errors: {state.errors}"
)
@@ -7,6 +7,9 @@ from unittest.mock import Mock
import pytest
from vllm import LLM, SamplingParams
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
from vllm.inputs import TextPrompt
from vllm.outputs import RequestOutput
def _make_mock_llm() -> LLM:
@@ -17,12 +20,13 @@ def _make_mock_llm() -> LLM:
return llm
def test_generate_forwards_mm_processor_kwargs() -> None:
def test_generate_forwards_mm_processor_kwargs(monkeypatch: pytest.MonkeyPatch) -> None:
llm = _make_mock_llm()
mm_processor_kwargs = {"num_crops": 4}
sampling_params = SamplingParams(max_tokens=1)
llm._run_completion = Mock(return_value=["ok"])
_run_completion = Mock(return_value=["ok"])
monkeypatch.setattr(llm, "_run_completion", _run_completion)
outputs = llm.generate(
"prompt",
@@ -31,17 +35,18 @@ def test_generate_forwards_mm_processor_kwargs() -> None:
)
assert outputs == ["ok"]
assert llm._run_completion.call_args.kwargs["mm_processor_kwargs"] == (
assert _run_completion.call_args.kwargs["mm_processor_kwargs"] == (
mm_processor_kwargs
)
def test_enqueue_forwards_mm_processor_kwargs() -> None:
def test_enqueue_forwards_mm_processor_kwargs(monkeypatch: pytest.MonkeyPatch) -> None:
llm = _make_mock_llm()
mm_processor_kwargs = {"do_resize": False}
sampling_params = SamplingParams(max_tokens=1)
llm._add_completion_requests = Mock(return_value=["req-0"])
_add_completion_requests = Mock(return_value=["req-0"])
monkeypatch.setattr(llm, "_add_completion_requests", _add_completion_requests)
request_ids = llm.enqueue(
"prompt",
@@ -51,18 +56,19 @@ def test_enqueue_forwards_mm_processor_kwargs() -> None:
)
assert request_ids == ["req-0"]
assert llm._add_completion_requests.call_args.kwargs["mm_processor_kwargs"] == (
assert _add_completion_requests.call_args.kwargs["mm_processor_kwargs"] == (
mm_processor_kwargs
)
def test_chat_forwards_mm_processor_kwargs() -> None:
def test_chat_forwards_mm_processor_kwargs(monkeypatch: pytest.MonkeyPatch) -> None:
llm = _make_mock_llm()
mm_processor_kwargs = {"do_pan_and_scan": True}
sampling_params = SamplingParams(max_tokens=1)
messages = [{"role": "user", "content": "hello"}]
messages: list[ChatCompletionMessageParam] = [{"role": "user", "content": "hello"}]
llm._run_chat = Mock(return_value=["ok"])
_run_chat = Mock(return_value=["ok"])
monkeypatch.setattr(llm, "_run_chat", _run_chat)
outputs = llm.chat(
messages,
@@ -71,18 +77,19 @@ def test_chat_forwards_mm_processor_kwargs() -> None:
)
assert outputs == ["ok"]
assert llm._run_chat.call_args.kwargs["mm_processor_kwargs"] == (
mm_processor_kwargs
)
assert _run_chat.call_args.kwargs["mm_processor_kwargs"] == mm_processor_kwargs
def test_enqueue_chat_forwards_mm_processor_kwargs() -> None:
def test_enqueue_chat_forwards_mm_processor_kwargs(
monkeypatch: pytest.MonkeyPatch,
) -> None:
llm = _make_mock_llm()
mm_processor_kwargs = {"do_pan_and_scan": True}
sampling_params = SamplingParams(max_tokens=1)
messages = [{"role": "user", "content": "hello"}]
messages: list[ChatCompletionMessageParam] = [{"role": "user", "content": "hello"}]
llm._add_chat_requests = Mock(return_value=["req-0"])
_add_chat_requests = Mock(return_value=["req-0"])
monkeypatch.setattr(llm, "_add_chat_requests", _add_chat_requests)
request_ids = llm.enqueue_chat(
messages,
@@ -92,67 +99,79 @@ def test_enqueue_chat_forwards_mm_processor_kwargs() -> None:
)
assert request_ids == ["req-0"]
assert llm._add_chat_requests.call_args.kwargs["mm_processor_kwargs"] == (
assert _add_chat_requests.call_args.kwargs["mm_processor_kwargs"] == (
mm_processor_kwargs
)
def test_run_chat_forwards_mm_processor_kwargs() -> None:
def test_run_chat_forwards_mm_processor_kwargs(monkeypatch: pytest.MonkeyPatch) -> None:
llm = _make_mock_llm()
mm_processor_kwargs = {"num_crops": 8}
sampling_params = SamplingParams(max_tokens=1)
messages = [{"role": "user", "content": "hello"}]
messages: list[ChatCompletionMessageParam] = [{"role": "user", "content": "hello"}]
sentinel_output = ["done"]
llm._add_chat_requests = Mock()
llm._run_engine = Mock(return_value=sentinel_output)
_add_chat_requests = Mock()
monkeypatch.setattr(llm, "_add_chat_requests", _add_chat_requests)
_run_engine = Mock(return_value=sentinel_output)
monkeypatch.setattr(llm, "_run_engine", _run_engine)
outputs = llm._run_chat(
messages=messages,
params=sampling_params,
output_type=object,
output_type=RequestOutput,
use_tqdm=False,
mm_processor_kwargs=mm_processor_kwargs,
)
assert outputs == sentinel_output
assert llm._add_chat_requests.call_args.kwargs["mm_processor_kwargs"] == (
assert _add_chat_requests.call_args.kwargs["mm_processor_kwargs"] == (
mm_processor_kwargs
)
def test_run_completion_forwards_mm_processor_kwargs() -> None:
def test_run_completion_forwards_mm_processor_kwargs(
monkeypatch: pytest.MonkeyPatch,
) -> None:
llm = _make_mock_llm()
mm_processor_kwargs = {"min_pixels": 4 * 28 * 28}
sampling_params = SamplingParams(max_tokens=1)
sentinel_output = ["done"]
llm._add_completion_requests = Mock()
llm._run_engine = Mock(return_value=sentinel_output)
_add_completion_requests = Mock()
monkeypatch.setattr(llm, "_add_completion_requests", _add_completion_requests)
_run_engine = Mock(return_value=sentinel_output)
monkeypatch.setattr(llm, "_run_engine", _run_engine)
outputs = llm._run_completion(
prompts=["prompt"],
params=sampling_params,
output_type=object,
output_type=RequestOutput,
use_tqdm=False,
mm_processor_kwargs=mm_processor_kwargs,
)
assert outputs == sentinel_output
assert llm._add_completion_requests.call_args.kwargs["mm_processor_kwargs"] == (
assert _add_completion_requests.call_args.kwargs["mm_processor_kwargs"] == (
mm_processor_kwargs
)
def test_add_completion_requests_forwards_mm_processor_kwargs() -> None:
def test_add_completion_requests_forwards_mm_processor_kwargs(
monkeypatch: pytest.MonkeyPatch,
) -> None:
llm = _make_mock_llm()
mm_processor_kwargs = {"max_dynamic_patch": 4}
sampling_params = SamplingParams(max_tokens=1)
llm._params_to_seq = Mock(return_value=[sampling_params])
llm._lora_request_to_seq = Mock(return_value=[None])
llm._priority_to_seq = Mock(return_value=[0])
llm._preprocess_cmpl_one = Mock(return_value={"prompt_token_ids": [1]})
_params_to_seq = Mock(return_value=[sampling_params])
monkeypatch.setattr(llm, "_params_to_seq", _params_to_seq)
_lora_request_to_seq = Mock(return_value=[None])
monkeypatch.setattr(llm, "_lora_request_to_seq", _lora_request_to_seq)
_priority_to_seq = Mock(return_value=[0])
monkeypatch.setattr(llm, "_priority_to_seq", _priority_to_seq)
_preprocess_cmpl_one = Mock(return_value={"prompt_token_ids": [1]})
monkeypatch.setattr(llm, "_preprocess_cmpl_one", _preprocess_cmpl_one)
captured_prompts = []
@@ -160,7 +179,8 @@ def test_add_completion_requests_forwards_mm_processor_kwargs() -> None:
captured_prompts.extend(prompts)
return ["req-0"]
llm._render_and_add_requests = Mock(side_effect=fake_render_and_add_requests)
_render_and_add_requests = Mock(side_effect=fake_render_and_add_requests)
monkeypatch.setattr(llm, "_render_and_add_requests", _render_and_add_requests)
request_ids = llm._add_completion_requests(
prompts=["prompt"],
@@ -170,7 +190,7 @@ def test_add_completion_requests_forwards_mm_processor_kwargs() -> None:
)
assert request_ids == ["req-0"]
llm._preprocess_cmpl_one.assert_called_once_with(
_preprocess_cmpl_one.assert_called_once_with(
"prompt",
None,
mm_processor_kwargs=mm_processor_kwargs,
@@ -183,7 +203,7 @@ def test_preprocess_cmpl_applies_mm_processor_kwargs_to_renderer(
) -> None:
llm = _make_mock_llm()
mm_processor_kwargs = {"num_crops": 8}
prompt = {"prompt": "<image>", "multi_modal_data": {"image": object()}}
prompt: TextPrompt = {"prompt": "<image>", "multi_modal_data": {"image": object()}}
renderer = Mock()
renderer.default_cmpl_tok_params = Mock()
@@ -213,7 +233,7 @@ def test_preprocess_cmpl_keeps_prompt_mm_processor_kwargs_when_no_override(
monkeypatch: pytest.MonkeyPatch,
) -> None:
llm = _make_mock_llm()
prompt = {
prompt: TextPrompt = {
"prompt": "<image>",
"multi_modal_data": {"image": object()},
"mm_processor_kwargs": {"num_crops": 2},
@@ -243,7 +263,9 @@ def test_preprocess_cmpl_keeps_prompt_mm_processor_kwargs_when_no_override(
def test_preprocess_chat_applies_mm_processor_kwargs_to_renderer() -> None:
llm = _make_mock_llm()
mm_processor_kwargs = {"num_crops": 8}
messages = [[{"role": "user", "content": "Describe this image."}]]
messages: list[list[ChatCompletionMessageParam]] = [
[{"role": "user", "content": "Describe this image."}]
]
renderer = Mock()
renderer.tokenizer = object()
@@ -269,7 +291,9 @@ def test_preprocess_chat_applies_mm_processor_kwargs_to_renderer() -> None:
def test_preprocess_chat_omits_mm_processor_kwargs_when_no_override() -> None:
llm = _make_mock_llm()
messages = [[{"role": "user", "content": "Describe this image."}]]
messages: list[list[ChatCompletionMessageParam]] = [
[{"role": "user", "content": "Describe this image."}]
]
renderer = Mock()
renderer.tokenizer = object()
@@ -291,7 +315,9 @@ def test_preprocess_chat_omits_mm_processor_kwargs_when_no_override() -> None:
def test_preprocess_chat_defaults_add_special_tokens_to_false() -> None:
# Matches `ChatCompletionRequest.add_special_tokens` on the server.
llm = _make_mock_llm()
messages = [[{"role": "user", "content": "hi"}]]
messages: list[list[ChatCompletionMessageParam]] = [
[{"role": "user", "content": "hi"}]
]
renderer = Mock()
renderer.tokenizer = object()
@@ -309,7 +335,9 @@ def test_preprocess_chat_defaults_add_special_tokens_to_false() -> None:
def test_preprocess_chat_tokenization_kwargs_override_add_special_tokens() -> None:
llm = _make_mock_llm()
messages = [[{"role": "user", "content": "hi"}]]
messages: list[list[ChatCompletionMessageParam]] = [
[{"role": "user", "content": "hi"}]
]
renderer = Mock()
renderer.tokenizer = object()
@@ -47,18 +47,20 @@ async def client(server):
@pytest.fixture(scope="session")
def base64_encoded_audio() -> dict[str, str]:
return {
audio_url: encode_audio_base64(*fetch_audio(audio_url))
for audio_url in TEST_AUDIO_URLS
}
encoded = {}
for audio_url in TEST_AUDIO_URLS:
audio, sample_rate = fetch_audio(audio_url)
encoded[audio_url] = encode_audio_base64(audio, int(sample_rate))
return encoded
@pytest.fixture(scope="session")
def url_encoded_audio() -> dict[str, str]:
return {
audio_url: encode_audio_url(*fetch_audio(audio_url))
for audio_url in TEST_AUDIO_URLS
}
encoded = {}
for audio_url in TEST_AUDIO_URLS:
audio, sample_rate = fetch_audio(audio_url)
encoded[audio_url] = encode_audio_url(audio, int(sample_rate))
return encoded
def dummy_messages_from_audio_url(
@@ -246,11 +246,15 @@ async def test_batched_chat_completions_logprob_token_ids(
}
def _make_request_output(prompt_idx: int, text: str) -> RequestOutput:
def _make_request_output(
prompt_idx: int,
text: str,
prompt_token_ids: list[int] | None = None,
) -> RequestOutput:
return RequestOutput(
request_id=f"req-{prompt_idx}",
prompt=None,
prompt_token_ids=[1, 2, 3],
prompt_token_ids=prompt_token_ids or [1, 2, 3],
prompt_logprobs=None,
outputs=[
CompletionOutput(
@@ -266,8 +270,12 @@ def _make_request_output(prompt_idx: int, text: str) -> RequestOutput:
)
async def _generator(prompt_idx: int, text: str) -> AsyncGenerator[RequestOutput, None]:
yield _make_request_output(prompt_idx, text)
async def _generator(
prompt_idx: int,
text: str,
prompt_token_ids: list[int] | None = None,
) -> AsyncGenerator[RequestOutput, None]:
yield _make_request_output(prompt_idx, text, prompt_token_ids)
@pytest.mark.asyncio
@@ -402,3 +410,47 @@ def test_batch_rejects_kv_transfer_prompt_token_ids():
],
kv_transfer_params={"prompt_token_ids": [10, 20, 30]},
)
@pytest.mark.asyncio
@pytest.mark.skip_global_cleanup
async def test_batched_parser_receives_each_prompt_prefix() -> None:
prefixes = []
class RecordingParser:
def __init__(self, *args, **kwargs):
self.prompt_token_ids = None
def set_prompt_token_ids(self, prompt_token_ids):
self.prompt_token_ids = prompt_token_ids
def parse(self, model_output, **kwargs):
prefixes.append(self.prompt_token_ids)
return None, model_output, None
serving = OpenAIServingChatBatch.__new__(OpenAIServingChatBatch)
serving.response_role = "assistant"
serving.system_fingerprint = None
serving.chat_template = None
serving.chat_template_content_format = "auto"
serving.default_chat_template_kwargs = {}
request = BatchChatCompletionRequest(
model="test-model",
messages=[
[{"role": "user", "content": "first"}],
[{"role": "user", "content": "second"}],
],
)
await serving.chat_completion_full_generator_batch(
request=request,
generators=[_generator(0, "one", [1]), _generator(1, "two", [2])],
request_id="req-prefix",
model_name="test-model",
all_conversations=request.messages,
tokenizer=object(),
request_metadata=RequestResponseMetadata(request_id="req-prefix"),
parser_cls=RecordingParser,
)
assert prefixes == [[1], [2]]
@@ -564,6 +564,9 @@ class MockModelConfig:
def get_diff_sampling_param(self):
return self.diff_sampling_param or {}
def get_vocab_size(self) -> int:
return 50257
@dataclass
class MockParallelConfig:
@@ -787,6 +790,64 @@ async def test_chat_per_request_metrics_suppressed_for_n_greater_than_one():
assert response.metrics is None
@pytest.mark.asyncio
async def test_chat_full_generator_gives_each_choice_parser_the_prompt():
prompts: list[list[int] | None] = []
class RecordingParser:
tool_parser_cls = None
def __init__(self, *args, **kwargs):
self.prompt_token_ids = None
def set_prompt_token_ids(self, prompt_token_ids):
self.prompt_token_ids = list(prompt_token_ids)
def parse(self, model_output, request, **kwargs):
prompts.append(self.prompt_token_ids)
return None, model_output, None
def count_reasoning_tokens(self, token_ids):
return 0
serving = _build_minimal_metrics_serving_chat(enable_per_request_metrics=False)
serving.parser_cls = RecordingParser
serving.model_config = None
serving.chat_template = None
serving.chat_template_content_format = "auto"
serving.default_chat_template_kwargs = {}
request_output = _make_metrics_request_output()
request_output.prompt_token_ids = [7, 8]
request_output.outputs.append(
CompletionOutput(
index=1,
text="Hi",
token_ids=[102],
cumulative_logprob=None,
logprobs=None,
finish_reason="stop",
)
)
await serving.chat_completion_full_generator(
ChatCompletionRequest(
model="test-model",
messages=[{"role": "user", "content": "Test prompt"}],
max_tokens=10,
n=2,
),
_single_request_output(request_output),
"chatcmpl-test-id",
"test-model",
conversation=[{"role": "user", "content": "Test"}],
tokenizer=MagicMock(),
request_metadata=RequestResponseMetadata(request_id="chatcmpl-test-id"),
parser=RecordingParser(),
)
assert prompts == [[7, 8], [7, 8]]
@pytest.mark.asyncio
async def test_chat_streaming_metrics_ride_on_usage_chunk():
serving = _build_minimal_metrics_serving_chat(enable_per_request_metrics=True)
@@ -1605,6 +1666,21 @@ async def test_serving_chat_truncation_side_controls_prompt_truncation():
assert left_token_ids == full_token_ids[-4:]
def test_serving_chat_starts_without_tokenizer():
"""Regression test: building the serving objects must not require a
tokenizer, which is absent under `skip_tokenizer_init`."""
mock_engine = _build_mock_engine()
mock_engine.model_config = MockModelConfig(skip_tokenizer_init=True)
mock_engine.renderer = HfRenderer(
MockVllmConfig(mock_engine.model_config, parallel_config=MockParallelConfig()),
None,
)
serving_chat = _build_serving_chat(mock_engine)
assert serving_chat.parser_cls is None
@pytest.mark.asyncio
async def test_serving_chat_mistral_token_ids_prompt_is_validated():
"""Regression test: when the Mistral tokenizer path returns token IDs
@@ -2860,7 +2936,16 @@ async def test_chat_kv_transfer_prompt_token_ids_rejects_invalid_ids(ids):
@pytest.mark.asyncio
async def test_chat_kv_transfer_prompt_token_ids_rejection_notifies_kv_connector():
@pytest.mark.parametrize(
("ids", "stream", "match"),
[
([1.5], False, "non-negative integers"),
([999999], True, r"out of vocabulary.*kv_transfer_params\.prompt_token_ids"),
],
)
async def test_chat_kv_transfer_prompt_token_ids_rejection_notifies_kv_connector(
ids, stream, match
):
"""A render-time 400 on a decode request notifies the KV connector."""
mock_engine = _build_mock_engine()
mock_engine.notify_kv_transfer_request_rejected = AsyncMock()
@@ -2870,9 +2955,10 @@ async def test_chat_kv_transfer_prompt_token_ids_rejection_notifies_kv_connector
request = ChatCompletionRequest(
model=MODEL_NAME,
messages=[{"role": "user", "content": "hi"}],
kv_transfer_params={"prompt_token_ids": [1.5], "do_remote_prefill": True},
stream=stream,
kv_transfer_params={"prompt_token_ids": ids, "do_remote_prefill": True},
)
with pytest.raises(VLLMValidationError, match="non-negative integers"):
with pytest.raises(VLLMValidationError, match=match):
await serving_chat.create_chat_completion(request)
mock_engine.notify_kv_transfer_request_rejected.assert_awaited_once_with(
@@ -37,7 +37,6 @@ def pairs_of_event_types() -> dict[str, str]:
"response.content_part.done": "response.content_part.added",
"response.output_text.done": "response.output_text.delta",
"response.reasoning_text.done": "response.reasoning_text.delta",
"response.reasoning_part.done": "response.reasoning_part.added",
"response.mcp_call_arguments.done": "response.mcp_call_arguments.delta",
"response.mcp_call.completed": "response.mcp_call.in_progress",
"response.function_call_arguments.done": "response.function_call_arguments.delta", # noqa: E501
@@ -126,6 +125,11 @@ def _validate_event_pairing(events: list, pairs_of_event_types: dict[str, str])
etype = event.type
if etype in end_events:
expected_start = pairs_of_event_types[etype]
# Deltas are optional, e.g. a function call with no arguments.
if expected_start.endswith(".delta") and (
not stack or stack[-1] != expected_start
):
continue
assert stack and stack[-1] == expected_start, (
f"Stack mismatch for {etype}: "
f"expected {expected_start}, "
@@ -245,11 +249,8 @@ def _validate_field_consistency(events: list) -> None:
active_content_index = None
continue
# --- content_part / reasoning_part added: sets content_index
if etype in (
"response.content_part.added",
"response.reasoning_part.added",
):
# --- content_part added: sets content_index
if etype == "response.content_part.added":
_assert_item_fields(event, etype, active_item_id, active_output_index)
content_index = getattr(event, "content_index", None)
if active_content_index is None:
@@ -340,12 +340,12 @@ async def test_function_calling_with_streaming_types(
# response.created
# -> response.in_progress
# -> response.output_item.added
# -> response.reasoning_part.added
# -> response.content_part.added
# -> response.reasoning_text.delta
# ....
# -> response.reasoning_text.delta
# -> response.reasoning_text.done
# -> response.reasoning_part.done
# -> response.content_part.done
# -> response.output_item.done
# -> response.output_item.added
# -> response.content_part.added
@@ -368,12 +368,12 @@ async def test_function_calling_with_streaming_types(
# response.created
# -> response.in_progress
# -> response.output_item.added
# -> response.reasoning_part.added
# -> response.content_part.added
# -> response.reasoning_text.delta
# ....
# -> response.reasoning_text.delta
# -> response.reasoning_text.done
# -> response.reasoning_part.done
# -> response.content_part.done
# -> response.output_item.done
# -> response.output_item.added
# -> response.content_part.added
@@ -424,7 +424,6 @@ async def test_function_calling_with_streaming_types(
"response.output_text.done": "response.output_text.delta",
"response.content_part.done": "response.content_part.added",
"response.reasoning_text.done": "response.reasoning_text.delta",
"response.reasoning_part.done": "response.reasoning_part.added",
"response.function_call_arguments.done": "response.function_call_arguments.delta", # noqa
}
@@ -467,10 +467,7 @@ async def test_streaming(client: OpenAI, model_name: str, background: bool):
assert event.item_id == current_item_id
# Verify content indices
if event.type in [
"response.content_part.added",
"response.reasoning_part.added",
]:
if event.type == "response.content_part.added":
assert event.content_index == current_content_index + 1
current_content_index = event.content_index
elif event.type in [
@@ -6,10 +6,13 @@ These tests verify that ParsableContext correctly delegates to the unified
Parser (via parse) and properly builds response output items.
"""
import json
from collections.abc import Sequence
from unittest.mock import MagicMock
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock
import pytest
from openai.types.responses import ResponseFunctionToolCall
from vllm.entrypoints.generate.base.protocol import (
DeltaMessage,
@@ -17,6 +20,7 @@ from vllm.entrypoints.generate.base.protocol import (
FunctionCall,
ToolCall,
)
from vllm.entrypoints.mcp.tool import HarmonyPythonTool
from vllm.entrypoints.openai.responses.context import ParsableContext
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.outputs import CompletionOutput, RequestOutput
@@ -317,6 +321,65 @@ def test_process_extracts_tool_calls():
assert tool_item.status == "completed"
@pytest.mark.parametrize(
("tool_name", "session_name", "dispatched_name", "arguments"),
[
("code_interpreter", "python", "python", {"code": "print(42)"}),
("web_search_preview", "browser", "search", {"query": "vLLM"}),
("container.exec", "container", "exec", {"cmd": ["pwd"]}),
],
)
@pytest.mark.asyncio
async def test_builtin_tool_output_preserves_function_call_id(
tool_name, session_name, dispatched_name, arguments
):
"""Built-in tool outputs remain correlated with their originating call."""
tool_session = MagicMock()
tool_session.call_tool = AsyncMock(
return_value=SimpleNamespace(content=[SimpleNamespace(text="result")])
)
context = _make_context(None, available_tools=[session_name])
tool_call = ResponseFunctionToolCall(
id=f"fc_{session_name}",
call_id=f"call_{session_name}",
type="function_call",
name=tool_name,
arguments=json.dumps(arguments),
)
context.response_messages.append(tool_call)
context._tool_sessions[session_name] = tool_session
output = await context.call_tool()
tool_session.call_tool.assert_awaited_once_with(dispatched_name, arguments)
assert output[0].call_id == tool_call.call_id
@pytest.mark.asyncio
async def test_local_python_tool_output_preserves_function_call_id():
"""Local code-interpreter output remains correlated with its call."""
async def process(_):
yield SimpleNamespace(content=[SimpleNamespace(text="result")])
python_tool = object.__new__(HarmonyPythonTool)
python_tool.python_tool = MagicMock(process=process)
context = _make_context(None, available_tools=["python"])
tool_call = ResponseFunctionToolCall(
id="fc_python",
call_id="call_python",
type="function_call",
name="code_interpreter",
arguments='{"code": "print(42)"}',
)
context.response_messages.append(tool_call)
context._tool_sessions["python"] = python_tool
output = await context.call_tool()
assert output[0].call_id == tool_call.call_id
# ---------------------------------------------------------------------------
# Tests: finish_reason tracking
# ---------------------------------------------------------------------------
@@ -34,6 +34,7 @@ from openai_harmony import Message as OpenAIHarmonyMessage
from openai_harmony import Role
import vllm.envs as envs
from tests.entrypoints.openai.chat_completion.test_serving_chat import MockModelConfig
from vllm.entrypoints.generate.base.protocol import (
DeltaFunctionCall,
DeltaMessage,
@@ -64,6 +65,7 @@ from vllm.entrypoints.openai.responses.streaming_events import (
)
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.inputs import tokens_input
from vllm.logprobs import Logprob as SampleLogprob
from vllm.outputs import CompletionOutput, RequestOutput
from vllm.parser.harmony import Segment
from vllm.renderers import TokenizeParams
@@ -126,14 +128,6 @@ def test_serialize_message_pydantic_model_returns_dict() -> None:
assert serialized["message"] == "hello"
@pytest.fixture
def mock_serving_responses():
"""Create a mock OpenAIServingResponses instance."""
serving_responses = MagicMock(spec=OpenAIServingResponses)
serving_responses.tool_server = MagicMock(spec=ToolServer)
return serving_responses
@pytest.fixture
def mock_context():
"""Create a mock conversation context."""
@@ -301,6 +295,102 @@ async def test_online_renderer_renders_non_harmony_responses_with_explicit_histo
assert preprocess_kwargs["default_template_kwargs"]["request_default"] == "kept"
_REUSED_IDS = [10, 20, 30]
_IMAGE_PART = {
"type": "input_image",
"image_url": "https://example.com/a.png",
"detail": "auto",
}
async def _render_responses_with_reuse(request, previous_messages=None):
online_renderer = OnlineRenderer(
model_config=MockModelConfig(),
renderer=MagicMock(),
request_logger=None,
chat_template=None,
chat_template_content_format="auto",
)
render_chat = AsyncMock(return_value=([[]], [tokens_input([1, 2])]))
online_renderer.renderer.render_chat_async = render_chat
result = await online_renderer.render_responses(
request, previous_messages=previous_messages
)
assert not isinstance(result, ErrorResponse)
return render_chat, result.engine_input
@pytest.mark.asyncio
@pytest.mark.parametrize("ids", [_REUSED_IDS, [1.5]])
@pytest.mark.parametrize(
"request_input",
[
[
{
"role": "user",
"content": [{"type": "input_text", "text": "look"}, _IMAGE_PART],
}
],
[
{
"type": "function_call",
"call_id": "call_1",
"name": "shot",
"arguments": "{}",
},
{
"type": "function_call_output",
"call_id": "call_1",
"output": [_IMAGE_PART],
},
],
],
ids=["input_image", "tool_output_image"],
)
async def test_responses_kv_transfer_prompt_token_ids_ignored_with_non_text_input(
request_input, ids
):
request = ResponsesRequest(
input=request_input,
kv_transfer_params={"do_remote_prefill": True, "prompt_token_ids": ids},
)
render_chat, _ = await _render_responses_with_reuse(request)
render_chat.assert_awaited_once()
assert request.kv_transfer_params == {"do_remote_prefill": True}
@pytest.mark.asyncio
async def test_responses_kv_transfer_prompt_token_ids_ignored_with_media_in_history():
request = ResponsesRequest(
input="next", kv_transfer_params={"prompt_token_ids": _REUSED_IDS}
)
render_chat, _ = await _render_responses_with_reuse(
request, previous_messages=[{"role": "user", "content": [_IMAGE_PART]}]
)
render_chat.assert_awaited_once()
@pytest.mark.asyncio
async def test_responses_kv_transfer_prompt_token_ids_kept_with_text_input():
request = ResponsesRequest(
input=[
{"role": "user", "content": [{"type": "input_text", "text": "hi"}]},
{"role": "assistant", "content": "ok"},
],
kv_transfer_params={"prompt_token_ids": _REUSED_IDS},
)
_, engine_input = await _render_responses_with_reuse(request)
assert engine_input["prompt_token_ids"] == _REUSED_IDS
@pytest.mark.asyncio
async def test_online_renderer_rejects_untrusted_responses_chat_template():
renderer = _new_online_renderer()
@@ -1359,6 +1449,7 @@ def _make_simple_context_with_output(
def _make_serving_instance(
*,
reasoning_parser: str = "",
tool_parser: str | None = None,
enable_per_request_metrics: bool = False,
) -> OpenAIServingResponses:
engine_client = MagicMock()
@@ -1379,6 +1470,8 @@ def _make_serving_instance(
chat_template=None,
chat_template_content_format="auto",
reasoning_parser=reasoning_parser,
enable_auto_tools=tool_parser is not None,
tool_parser=tool_parser,
enable_per_request_metrics=enable_per_request_metrics,
)
@@ -1977,3 +2070,113 @@ class TestAutoToolStreaming:
assert len(function_done) == 1
assert function_done[0].item.name == "get_weather"
assert function_done[0].item.arguments == tool_args
@pytest.mark.skip_global_cleanup
@pytest.mark.asyncio
async def test_stream_completed_response_reuses_streamed_items(monkeypatch):
"""response.completed must carry the streamed items (same ids,
call_id, logprobs) rather than a reparse of the full output."""
monkeypatch.setattr(envs, "VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT", False)
serving = _make_serving_instance_with_reasoning()
call = DeltaToolCall(
id="chatcmpl-tool-parser-id",
index=0,
function=DeltaFunctionCall(name="get_weather", arguments="{}"),
)
response_parser = _mock_parser_with_reasoning(
serving,
[
DeltaMessage(reasoning="think"),
DeltaMessage(content="Hi"),
DeltaMessage(tool_calls=[call]),
],
)
response_parser.count_reasoning_tokens = MagicMock(return_value=1)
context = SimpleContext(response_parser=response_parser)
async def result_generator():
for token_id, text in [(10, "think"), (20, "Hi"), (30, "call")]:
output = _make_request_output(text, [token_id])
output.outputs[0].logprobs = [
{token_id: SampleLogprob(logprob=-0.5, decoded_token=text)}
]
context.append_output(output)
yield context
events = [
event
async for event in serving.responses_stream_generator(
request=ResponsesRequest(
input="hi",
include=["message.output_text.logprobs"],
stream=True,
store=False,
),
sampling_params=SamplingParams(max_tokens=16),
result_generator=result_generator(),
context=context,
model_name="test-model",
tokenizer=MagicMock(),
request_metadata=RequestResponseMetadata(request_id="req"),
)
]
streamed = [e.item for e in events if e.type == "response.output_item.done"]
assert events[-1].response.output == streamed
_, message, function_call = streamed
assert [lp.token for lp in message.content[0].logprobs] == ["Hi"]
assert function_call.call_id == "chatcmpl-tool-parser-id"
@pytest.mark.asyncio
@pytest.mark.parametrize(("parallel_tool_calls", "num_calls"), [(False, 1), (None, 3)])
async def test_streaming_parallel_tool_calls(
monkeypatch, parallel_tool_calls, num_calls
):
"""parallel_tool_calls=False keeps only the first call even when the grammar
does not limit the model; None keeps every call."""
monkeypatch.setattr(envs, "VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT", False)
serving = _make_serving_instance(tool_parser="hermes")
request = ResponsesRequest(
input="hi",
tools=[{"type": "function", "name": "get_weather", "parameters": {}}],
parallel_tool_calls=parallel_tool_calls,
stream=True,
)
context = SimpleContext(
response_parser=serving._make_response_parser(request, MagicMock(), {})
)
args = [f'{{"city": "{city}"}}' for city in ("Paris", "Berlin", "Tokyo")]
async def result_generator():
for i, arg in enumerate(args):
text = f'<tool_call>\n{{"name": "get_weather", "arguments": {arg}}}'
context.append_output(_make_request_output(f"{text}\n</tool_call>\n", [i]))
yield context
events = [
event
async for event in serving.responses_stream_generator(
request=request,
sampling_params=SamplingParams(max_tokens=64),
result_generator=result_generator(),
context=context,
model_name="test-model",
tokenizer=MagicMock(),
request_metadata=RequestResponseMetadata(request_id="req"),
)
]
assert [
event.output_index
for event in events
if event.type == "response.output_item.added"
] == list(range(num_calls))
assert all(getattr(event, "output_index", 0) < num_calls for event in events)
assert [
event.item.arguments
for event in events
if event.type == "response.output_item.done"
] == args[:num_calls]
assert [item.arguments for item in events[-1].response.output] == args[:num_calls]
@@ -1,8 +1,10 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from openai.types.responses import ResponseFunctionWebSearch
import pytest
from openai.types.responses import ResponseFunctionWebSearch, ResponseStreamEvent
from openai_harmony import Message, Role
from pydantic import TypeAdapter
from vllm.entrypoints.generate.base.protocol import (
DeltaFunctionCall,
@@ -14,6 +16,8 @@ from vllm.entrypoints.openai.responses.streaming_events import (
StreamingState,
_StateType,
emit_browser_tool_events,
emit_reasoning_delta_events,
emit_reasoning_done_events,
split_delta,
)
@@ -40,6 +44,41 @@ def test_browser_find_uses_responses_action_type():
assert event.item.action.url == "cursor:42"
@pytest.mark.parametrize("harmony", [False, True], ids=["simple", "harmony"])
def test_reasoning_content_parts_use_sdk_events(harmony):
text = "Let me think."
if harmony:
state = StreamingState()
events = emit_reasoning_delta_events(text, state)
events.extend(emit_reasoning_done_events(text, state))
else:
processor = SimpleStreamingEventProcessor()
events = _run_through_processor(processor, DeltaMessage(reasoning=text))
events.extend(processor.close_current())
assert [event.type for event in events] == [
"response.output_item.added",
"response.content_part.added",
"response.reasoning_text.delta",
"response.reasoning_text.done",
"response.content_part.done",
"response.output_item.done",
]
added, part_added, delta, text_done, part_done, done = events
assert part_added.part.type == part_done.part.type == "reasoning_text"
assert part_added.part.text == ""
assert delta.delta == text_done.text == part_done.part.text == text
assert done.item.content[0].text == text
assert done.item.id == added.item.id
for event in events[1:-1]:
assert event.item_id == added.item.id
assert event.content_index == part_added.content_index
validator = TypeAdapter(ResponseStreamEvent)
for event in events:
assert event.output_index == added.output_index
validator.validate_python(event.model_dump())
def _make_tool_call(
index: int, name: str | None = None, arguments: str | None = None
) -> DeltaToolCall:
@@ -225,7 +225,7 @@ def online_renderer(model_config: MockModelConfig, request) -> OnlineRenderer:
renderer.tokenizer = MagicMock()
return OnlineRenderer(
model_config=model_config, # type: ignore[arg-type]
model_config=model_config,
renderer=renderer,
request_logger=None,
chat_template=None,
@@ -7,6 +7,7 @@ import torch
from tests.models.utils import softmax
from vllm import LLM, ClassificationRequestOutput, PoolingParams
from vllm.inputs import PromptType
from vllm.tasks import PoolingTask
MODEL_NAME = "jason9693/Qwen2.5-1.5B-apeach"
@@ -53,7 +54,8 @@ def test_token_ids_prompts(llm: LLM):
@pytest.mark.skip_global_cleanup
def test_list_prompts(llm: LLM):
outputs = llm.classify([prompt, prompt_token_ids], use_tqdm=False)
prompts: list[PromptType] = [prompt, prompt_token_ids]
outputs = llm.classify(prompts, use_tqdm=False)
assert len(outputs) == 2
for i in range(len(outputs)):
assert isinstance(outputs[i], ClassificationRequestOutput)
@@ -21,6 +21,7 @@ from vllm.entrypoints.pooling.typing import (
PoolingEngineInput,
PoolingServeContext,
)
from vllm.inputs import tokens_input
from vllm.outputs import PoolingOutput, PoolingRequestOutput
@@ -28,7 +29,7 @@ class TestEmbeddingRequestParsing:
"""Unit tests for OpenAI embedding request parsing."""
def test_input_messages_parses_as_chat_request(self):
request = TypeAdapter(EmbeddingRequest).validate_python(
request: EmbeddingRequest = TypeAdapter(EmbeddingRequest).validate_python(
{
"model": "test",
"input": [{"role": "user", "content": "hello"}],
@@ -42,7 +43,7 @@ class TestEmbeddingRequestParsing:
assert request.chat_template_kwargs == {"instruction": "Represent the query: "}
def test_batched_input_messages_parses_as_batch_chat_input_request(self):
request = TypeAdapter(EmbeddingRequest).validate_python(
request: EmbeddingRequest = TypeAdapter(EmbeddingRequest).validate_python(
{
"model": "test",
"input": [
@@ -65,7 +66,7 @@ class TestEmbeddingRequestParsing:
assert request.chat_template_kwargs == {"instruction": "Represent the query: "}
def test_token_ids_still_parse_as_completion_request(self):
request = TypeAdapter(EmbeddingRequest).validate_python(
request: EmbeddingRequest = TypeAdapter(EmbeddingRequest).validate_python(
{
"model": "test",
"input": [[1, 2, 3], [4, 5]],
@@ -76,7 +77,7 @@ class TestEmbeddingRequestParsing:
assert request.input == [[1, 2, 3], [4, 5]]
def test_messages_still_parses_as_chat_request(self):
request = TypeAdapter(EmbeddingRequest).validate_python(
request: EmbeddingRequest = TypeAdapter(EmbeddingRequest).validate_python(
{
"model": "test",
"messages": [{"role": "user", "content": "hello"}],
@@ -89,7 +90,7 @@ class TestEmbeddingRequestParsing:
assert request.chat_template_kwargs == {"instruction": "Represent the query: "}
def test_batched_messages_parses_as_batch_chat_request(self):
request = TypeAdapter(EmbeddingRequest).validate_python(
request: EmbeddingRequest = TypeAdapter(EmbeddingRequest).validate_python(
{
"model": "test",
"messages": [
@@ -416,7 +417,7 @@ class TestChunkedEmbeddingProcessing:
@staticmethod
def _make_context() -> PoolingServeContext[EmbeddingCompletionRequest]:
request = TypeAdapter(EmbeddingRequest).validate_python(
request: EmbeddingRequest = TypeAdapter(EmbeddingRequest).validate_python(
{
"model": "test",
"input": [[0, 1, 2, 3, 4], [10, 11]],
@@ -431,13 +432,13 @@ class TestChunkedEmbeddingProcessing:
request_id="embd-client-prompt-999-chunk-888",
engine_inputs=[
PoolingEngineInput(
prompts={"prompt_token_ids": [0, 1, 2, 3, 4]},
prompts=tokens_input(prompt_token_ids=[0, 1, 2, 3, 4]),
params=pooling_params,
lora_requests=None,
priorities=0,
),
PoolingEngineInput(
prompts={"prompt_token_ids": [10, 11]},
prompts=tokens_input(prompt_token_ids=[10, 11]),
params=pooling_params,
lora_requests=None,
priorities=0,
@@ -7,6 +7,7 @@ import torch
import torch.nn.functional as F
from vllm import LLM, EmbeddingRequestOutput, PoolingParams
from vllm.inputs import PromptType
from vllm.tasks import PoolingTask
MODEL_NAME = "intfloat/multilingual-e5-small"
@@ -54,7 +55,8 @@ def test_token_ids_prompts(llm: LLM):
@pytest.mark.skip_global_cleanup
def test_list_prompts(llm: LLM):
outputs = llm.embed([prompt, prompt_token_ids], use_tqdm=False)
prompts: list[PromptType] = [prompt, prompt_token_ids]
outputs = llm.embed(prompts, use_tqdm=False)
assert len(outputs) == 2
for i in range(len(outputs)):
assert isinstance(outputs[i], EmbeddingRequestOutput)
+13 -9
View File
@@ -12,9 +12,13 @@ import requests
import torch
import torch.nn.functional as F
from tests.models.language.pooling.embed_utils import run_embedding_correctness_test
from tests.models.language.pooling.embed_utils import (
float_embeddings,
run_embedding_correctness_test,
)
from tests.models.utils import check_embeddings_close
from tests.utils import RemoteOpenAIServer
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
from vllm.entrypoints.pooling.embed.protocol import EmbeddingResponse
from vllm.entrypoints.pooling.pooling.protocol import PoolingResponse
from vllm.entrypoints.pooling.utils import (
@@ -145,7 +149,7 @@ async def test_completion_request(
assert embeddings.usage.prompt_tokens == len(input_tokens)
assert embeddings.usage.total_tokens == len(input_tokens)
vllm_outputs = [d.embedding for d in embeddings.data]
vllm_outputs = float_embeddings(embeddings)
run_embedding_correctness_test(hf_model, [input_text], vllm_outputs)
# test input: list[int]
@@ -165,7 +169,7 @@ async def test_completion_request(
assert embeddings.usage.prompt_tokens == len(input_tokens)
assert embeddings.usage.total_tokens == len(input_tokens)
vllm_outputs = [d.embedding for d in embeddings.data]
vllm_outputs = float_embeddings(embeddings)
run_embedding_correctness_test(hf_model, [input_text], vllm_outputs)
@@ -194,7 +198,7 @@ async def test_completion_request_batched(
assert embeddings.usage.prompt_tokens == len(input_tokens) * N
assert embeddings.usage.total_tokens == len(input_tokens) * N
vllm_outputs = [d.embedding for d in embeddings.data]
vllm_outputs = float_embeddings(embeddings)
run_embedding_correctness_test(hf_model, input_texts, vllm_outputs)
# test list[list[int]]
@@ -214,7 +218,7 @@ async def test_completion_request_batched(
assert embeddings.usage.prompt_tokens == len(input_tokens) * N
assert embeddings.usage.total_tokens == len(input_tokens) * N
vllm_outputs = [d.embedding for d in embeddings.data]
vllm_outputs = float_embeddings(embeddings)
run_embedding_correctness_test(hf_model, input_texts, vllm_outputs)
@@ -331,7 +335,7 @@ async def test_padding(client: openai.AsyncOpenAI, model_name: str):
async def test_chat_request(
server: RemoteOpenAIServer, client: openai.AsyncOpenAI, model_name: str
):
messages = [
messages: list[ChatCompletionMessageParam] = [
{
"role": "user",
"content": "The cat sat on the mat.",
@@ -382,8 +386,8 @@ async def test_chat_request(
assert chat_embeddings.created <= completion_embeddings.created
# Use tolerance-based comparison for embeddings
check_embeddings_close(
embeddings_0_lst=[d.embedding for d in chat_embeddings.data],
embeddings_1_lst=[d.embedding for d in completion_embeddings.data],
embeddings_0_lst=float_embeddings(chat_embeddings),
embeddings_1_lst=float_embeddings(completion_embeddings),
name_0="chat",
name_1="completion",
)
@@ -488,7 +492,7 @@ async def test_invocations_completion_request(
@pytest.mark.asyncio
async def test_invocations_chat_request(server: RemoteOpenAIServer):
messages = [
messages: list[ChatCompletionMessageParam] = [
{
"role": "user",
"content": "The cat sat on the mat.",
@@ -6,7 +6,10 @@ import openai
import pytest
from tests.conftest import HfRunner
from tests.models.language.pooling.embed_utils import run_embedding_correctness_test
from tests.models.language.pooling.embed_utils import (
float_embeddings,
run_embedding_correctness_test,
)
from tests.models.utils import EmbedModelInfo
from tests.utils import ROCM_EXTRA_ARGS, RemoteOpenAIServer
from vllm.entrypoints.pooling.embed.protocol import EmbeddingResponse
@@ -95,7 +98,7 @@ async def test_matryoshka(
if dimensions is not None:
assert len(embeddings.data[0].embedding) == dimensions
vllm_outputs = [d.embedding for d in embeddings.data]
vllm_outputs = float_embeddings(embeddings)
run_embedding_correctness_test(hf_model, prompts, vllm_outputs, dimensions)
if model_info.is_matryoshka:
@@ -274,6 +274,7 @@ async def test_long_text_embedding_1500_chars(
# Verify embedding vector validity
embedding_vector = embeddings.data[0].embedding
assert isinstance(embedding_vector, list)
assert all(isinstance(x, float) for x in embedding_vector), (
"Embedding vector should contain floats"
)
@@ -322,6 +323,7 @@ async def test_long_text_embedding_2500_chars(
# Verify embedding vector validity
embedding_vector = embeddings.data[0].embedding
assert isinstance(embedding_vector, list)
assert all(isinstance(x, float) for x in embedding_vector), (
"Embedding vector should contain floats"
)
@@ -409,6 +411,7 @@ async def test_chunked_vs_normal_consistency(
# 验证embedding向量的有效性
embedding_vector = embeddings.data[0].embedding
assert isinstance(embedding_vector, list)
assert all(isinstance(x, float) for x in embedding_vector)
assert not all(x == 0 for x in embedding_vector)
@@ -438,6 +441,7 @@ async def test_chunked_processing_response_format(
# Verify embedding vector properties
embedding_vector = embeddings.data[0].embedding
assert isinstance(embedding_vector, list)
import math
vector_norm = math.sqrt(sum(x * x for x in embedding_vector))
@@ -11,6 +11,7 @@ import torch
from tests.models.utils import check_embeddings_close
from tests.utils import RemoteOpenAIServer
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
from vllm.entrypoints.pooling.pooling.protocol import PoolingResponse
from vllm.entrypoints.pooling.utils import (
MetadataItem,
@@ -137,7 +138,7 @@ def test_completion_request_batched(server: RemoteOpenAIServer, model_name: str)
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
async def test_chat_request(server: RemoteOpenAIServer, model_name: str):
messages = [
messages: list[ChatCompletionMessageParam] = [
{
"role": "user",
"content": "The cat sat on the mat.",
@@ -524,7 +525,7 @@ async def test_invocations_chat_request(server: RemoteOpenAIServer):
@pytest.mark.asyncio
async def test_invocations_conversation_chat_request(server: RemoteOpenAIServer):
messages = [
messages: list[ChatCompletionMessageParam] = [
{
"role": "user",
"content": "The cat sat on the mat.",
@@ -95,12 +95,16 @@ def test_token_type_ids_stay_aligned_with_a_truncated_padded_prompt():
truncation_side="left",
)
prompt = tok_params.apply_post_tokenization(tokenizer, prompt)
processed_prompt = tok_params.apply_post_tokenization(
tokenizer,
prompt,
)
assert "prompt_token_ids" in processed_prompt
token_type_ids = _apply_post_tokenization_to_token_type_ids(
tokenizer, tok_params, token_type_ids
)
prompt_token_ids = prompt["prompt_token_ids"]
prompt_token_ids = processed_prompt["prompt_token_ids"]
assert len(token_type_ids) == len(prompt_token_ids)
# Keeping the last 40 tokens drops the first 10 query tokens, so 10 query
@@ -30,12 +30,18 @@ def test_online_forwards_truncate_prompt_tokens_to_proxy(monkeypatch):
silently disables truncate_prompt_tokens for Jina rerank/score.
"""
proc = JinaRankingIOProcessor.__new__(JinaRankingIOProcessor)
proc.valid_inputs_online = MagicMock(
return_value=ScoringData(data_1=["query"], data_2=["doc"])
monkeypatch.setattr(
proc,
"valid_inputs_online",
MagicMock(return_value=ScoringData(data_1=["query"], data_2=["doc"])),
)
monkeypatch.setattr(proc, "_get_token_limits", MagicMock(return_value=(0, 0)))
monkeypatch.setattr(
proc, "ensure_str", MagicMock(side_effect=lambda data: list(data))
)
monkeypatch.setattr(
proc, "format_docs_prompts_func", MagicMock(return_value="formatted prompt")
)
proc._get_token_limits = MagicMock(return_value=(0, 0))
proc.ensure_str = MagicMock(side_effect=lambda data: list(data))
proc.format_docs_prompts_func = MagicMock(return_value="formatted prompt")
captured: dict[str, object] = {}
@@ -6,7 +6,7 @@ from unittest.mock import AsyncMock, Mock
import pytest
from vllm.entrypoints.pooling.scoring.serving import ServingScores
from vllm.entrypoints.pooling.typing import PoolingServeContext
from vllm.entrypoints.pooling.typing import PoolingEngineInput, PoolingServeContext
from vllm.inputs import tokens_input
from vllm.pooling_params import PoolingParams
@@ -21,8 +21,18 @@ def _make_context() -> PoolingServeContext:
priorities=None,
prompt_extras=None,
engine_inputs=[
tokens_input(prompt_token_ids=[1]),
tokens_input(prompt_token_ids=[2]),
PoolingEngineInput(
prompts=tokens_input(prompt_token_ids=[1]),
params=PoolingParams(task="token_embed"),
lora_requests=None,
priorities=0,
),
PoolingEngineInput(
prompts=tokens_input(prompt_token_ids=[2]),
params=PoolingParams(task="token_embed"),
lora_requests=None,
priorities=0,
),
],
n_queries=1,
)
@@ -39,10 +49,13 @@ def _query_key(context: PoolingServeContext) -> str:
@pytest.mark.asyncio
async def test_colliding_request_ids_use_distinct_query_cache_keys():
async def test_colliding_request_ids_use_distinct_query_cache_keys(
monkeypatch: pytest.MonkeyPatch,
):
serving = object.__new__(ServingScores)
serving._prepare_generators = AsyncMock()
serving._collect_batch = AsyncMock()
prepare_generators = AsyncMock()
monkeypatch.setattr(serving, "_prepare_generators", prepare_generators)
monkeypatch.setattr(serving, "_collect_batch", AsyncMock())
first = _make_context()
second = _make_context()
@@ -50,9 +63,7 @@ async def test_colliding_request_ids_use_distinct_query_cache_keys():
await serving._flash_late_interaction_encode_queries(context)
await serving._flash_late_interaction_encode_docs(context)
prepared_contexts = [
call.args[0] for call in serving._prepare_generators.await_args_list
]
prepared_contexts = [call.args[0] for call in prepare_generators.await_args_list]
first_query_key = _query_key(prepared_contexts[0])
first_doc_key = _query_key(prepared_contexts[1])
second_query_key = _query_key(prepared_contexts[2])
@@ -45,6 +45,7 @@ async def test_pooling_token_classify(server: RemoteOpenAIServer, model_name: st
poolings = PoolingResponse.model_validate(response.json())
assert len(poolings.data) == 1
assert len(poolings.data[0].data) == 8
assert isinstance(poolings.data[0].data[0], list)
assert len(poolings.data[0].data[0]) == 2
@@ -71,6 +71,7 @@ async def test_pooling_token_embed(server: RemoteOpenAIServer, model_name: str):
assert poolings.id == f"pooling-{request_id}"
assert len(poolings.data) == 1
assert len(poolings.data[0].data) == len(input_tokens)
assert isinstance(poolings.data[0].data[0], list)
assert len(poolings.data[0].data[0]) == 384
@@ -27,6 +27,11 @@ async def client(server):
yield http_client
@pytest.fixture(scope="module")
def tokenizer():
return get_tokenizer(MODEL_NAME)
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
@@ -372,6 +377,58 @@ async def test_derender_chat_model_omitted_resolves_served_name(client):
assert response.json()["model"] == MODEL_NAME
@pytest.mark.asyncio
async def test_derender_chat_leading_space_seeded_from_prompt(client, tokenizer):
"""prompt_token_ids keeps the first token's leading space on this
Metaspace tokenizer, matching the coupled path."""
gen_req = await _render_chat(client)
prompt_token_ids = gen_req["token_ids"]
output_ids = tokenizer.encode("Hello there, output", add_special_tokens=False)
unseeded = await client.post(
"/v1/chat/completions/derender",
json={
"model": MODEL_NAME,
"generate_response": _make_generate_response(output_ids),
},
)
seeded = await client.post(
"/v1/chat/completions/derender",
json={
"model": MODEL_NAME,
"generate_response": _make_generate_response(output_ids),
"prompt_token_ids": prompt_token_ids,
},
)
assert unseeded.status_code == 200
assert seeded.status_code == 200
unseeded_content = unseeded.json()["choices"][0]["message"]["content"]
seeded_content = seeded.json()["choices"][0]["message"]["content"]
assert not unseeded_content.startswith(" ")
assert seeded_content.startswith(" ")
assert seeded_content.lstrip(" ") == unseeded_content
@pytest.mark.asyncio
async def test_derender_chat_oversized_prompt_token_ids_rejected(client):
"""prompt_token_ids longer than max_model_len returns 400."""
gen_req = await _render_chat(client)
synthetic_ids = gen_req["token_ids"][:3]
oversized_prompt_ids = [42] * 1_000_000
response = await client.post(
"/v1/chat/completions/derender",
json={
"model": MODEL_NAME,
"generate_response": _make_generate_response(synthetic_ids),
"prompt_token_ids": oversized_prompt_ids,
},
)
assert response.status_code == 400
assert "max_model_len" in response.json()["error"]["message"]
# ---------------------------------------------------------------------------
# Completion derender tests
# ---------------------------------------------------------------------------
@@ -486,6 +543,84 @@ async def test_derender_completion_prompt_tokens_length_mismatch(client):
assert response.status_code == 400
@pytest.mark.asyncio
async def test_derender_completion_prompt_token_ids_length_mismatch(client):
"""len(prompt_token_ids) != len(generate_responses) returns 400, the
same as the existing prompt_tokens length check."""
gr1 = await _render_completion(client, "Hello")
ids1 = gr1["token_ids"][:3]
response = await client.post(
"/v1/completions/derender",
json={
"model": MODEL_NAME,
"generate_responses": [
_make_completion_generate_response(ids1, gr1["request_id"]),
],
"prompt_token_ids": [[1, 2], [3, 4]],
},
)
assert response.status_code == 400
@pytest.mark.asyncio
async def test_derender_completion_leading_space_seeded_from_prompt(client, tokenizer):
"""Completions counterpart of
test_derender_chat_leading_space_seeded_from_prompt."""
gr1 = await _render_completion(client, "Hello world")
prompt_token_ids = gr1["token_ids"]
output_ids = tokenizer.encode("Hello there, output", add_special_tokens=False)
unseeded = await client.post(
"/v1/completions/derender",
json={
"model": MODEL_NAME,
"generate_responses": [
_make_completion_generate_response(output_ids, gr1["request_id"]),
],
},
)
seeded = await client.post(
"/v1/completions/derender",
json={
"model": MODEL_NAME,
"generate_responses": [
_make_completion_generate_response(output_ids, gr1["request_id"]),
],
"prompt_token_ids": [prompt_token_ids],
},
)
assert unseeded.status_code == 200
assert seeded.status_code == 200
unseeded_text = unseeded.json()["choices"][0]["text"]
seeded_text = seeded.json()["choices"][0]["text"]
assert not unseeded_text.startswith(" ")
assert seeded_text.startswith(" ")
assert seeded_text.lstrip(" ") == unseeded_text
@pytest.mark.asyncio
async def test_derender_completion_oversized_prompt_token_ids_rejected(client):
"""A prompt_token_ids entry longer than max_model_len returns 400."""
gr1 = await _render_completion(client, "Hello")
ids1 = gr1["token_ids"][:3]
oversized_prompt_ids = [42] * 1_000_000
response = await client.post(
"/v1/completions/derender",
json={
"model": MODEL_NAME,
"generate_responses": [
_make_completion_generate_response(ids1, gr1["request_id"]),
],
"prompt_token_ids": [oversized_prompt_ids],
},
)
assert response.status_code == 400
assert "max_model_len" in response.json()["error"]["message"]
@pytest.mark.asyncio
async def test_derender_completion_empty_generate_responses(client):
"""Empty generate_responses list returns 400."""
@@ -20,11 +20,17 @@ Tests are split into two layers:
import json
import threading
from concurrent.futures import ThreadPoolExecutor
from unittest.mock import AsyncMock, MagicMock
import httpx
import pytest
import pytest_asyncio
from pydantic import ValidationError
from tokenizers.decoders import DecodeStream
import vllm.renderers.online_derenderer as online_derenderer_module
from tests.entrypoints.scale_out.derender.utils import stream_chat_derender
from tests.utils import RemoteLaunchRenderServer
from vllm.entrypoints.generate.base.protocol import (
DeltaFunctionCall,
DeltaMessage,
@@ -32,7 +38,17 @@ from vllm.entrypoints.generate.base.protocol import (
PerRequestMetrics,
SpeculativeDecodingMetrics,
)
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
ChatCompletionStreamResponse,
)
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
from vllm.entrypoints.scale_out.derender.serving import ServingDerender
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
DerenderChatRequest,
DerenderChatStreamRequest,
DerenderCompletionRequest,
DerenderCompletionStreamRequest,
DerenderStreamState,
GenerateResponse,
GenerateResponseChoice,
@@ -41,6 +57,12 @@ from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
)
from vllm.entrypoints.serve.engine.protocol import ErrorResponse, UsageInfo
from vllm.parser import Parser
from vllm.renderers.online_derenderer import (
OnlineDerenderer,
_decode_params,
_seed_stream_state,
)
from vllm.tokenizers import get_tokenizer
from vllm.utils import random_uuid
MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
@@ -169,6 +191,8 @@ def _make_stream_chunk(
request_id: str = "test-req",
usage: dict | None = None,
metrics: PerRequestMetrics | None = None,
logprobs: dict | None = None,
prompt_token_ids: list[int] | None = None,
) -> GenerateStreamResponse:
"""Build a GenerateStreamResponse SSE chunk."""
return GenerateStreamResponse(
@@ -178,13 +202,31 @@ def _make_stream_chunk(
index=index,
token_ids=token_ids,
finish_reason=finish_reason,
logprobs=logprobs,
)
],
usage=UsageInfo(**usage) if usage else None,
metrics=metrics,
prompt_token_ids=prompt_token_ids,
)
def _placeholder_logprobs(token_ids: list[int]) -> dict:
"""Per-token logprob entries using token_id:N placeholders, as sent by
the generate worker."""
return {
"content": [
{
"token": f"token_id:{tid}",
"logprob": -0.5,
"bytes": None,
"top_logprobs": [],
}
for tid in token_ids
]
}
def _make_usage_chunk(
completion_tokens: int,
prompt_tokens: int = 0,
@@ -210,18 +252,12 @@ def _make_usage_chunk(
@pytest.fixture(scope="module")
def tokenizer():
"""Load the tiny tokenizer used across unit tests."""
from vllm.tokenizers import get_tokenizer
return get_tokenizer(MODEL_NAME)
@pytest.fixture(scope="module")
def derenderer(tokenizer, request):
"""Construct a minimal OnlineDerenderer backed by a stub renderer."""
from unittest.mock import MagicMock
from vllm.renderers.online_derenderer import OnlineDerenderer
renderer = MagicMock()
renderer.get_tokenizer.return_value = tokenizer
@@ -255,10 +291,6 @@ def parsed_derenderer(tokenizer):
`renderer._executor` must be a real `ThreadPoolExecutor`.
`loop.run_in_executor` cannot submit work to a `MagicMock`.
"""
from unittest.mock import MagicMock
from vllm.renderers.online_derenderer import OnlineDerenderer
renderer = MagicMock()
renderer.get_tokenizer.return_value = tokenizer
renderer._executor = ThreadPoolExecutor(max_workers=2)
@@ -278,6 +310,45 @@ def parsed_derenderer(tokenizer):
return dr
def test_non_streaming_derender_initializes_parser_prefix(
derenderer, tokenizer, monkeypatch
):
from unittest.mock import MagicMock
from vllm.entrypoints.generate.base.protocol import FunctionCall
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
parser = MagicMock()
parser.parse.return_value = (
None,
None,
[FunctionCall(name="tool", arguments="{}")],
)
monkeypatch.setattr(derenderer, "parser", MagicMock(return_value=parser))
generated_ids = tokenizer.encode("answer", add_special_tokens=False)
response = GenerateResponse(
request_id="test",
prompt_token_ids=[11, 12],
choices=[
GenerateResponseChoice(
index=0,
token_ids=generated_ids,
finish_reason="length",
)
],
)
request = ChatCompletionRequest(
model=MODEL_NAME,
messages=[{"role": "user", "content": "question"}],
)
choices = derenderer._derender_chat(response, request)
parser.set_prompt_token_ids.assert_called_once_with([11, 12])
assert choices[0].message.tool_calls[0].function.name == "tool"
assert choices[0].finish_reason == "length"
class TestDetokenizeDelta:
"""_detokenize_delta: chunked decode must equal one shot decode."""
@@ -376,6 +447,57 @@ class TestDetokenizeDelta:
# Bounded by a small constant independent of len(token_ids)
assert max_window <= 32
def test_truncated_multibyte_char_is_held_back(self, derenderer, tokenizer):
"""Output ending mid-character emits no U+FFFD: batch == stream == engine."""
emoji_ids = tokenizer.encode("🌈", add_special_tokens=False)
byte_ids = emoji_ids[-4:]
assert len(byte_ids) == 4
output_ids = byte_ids[:-1]
batch, _ = derenderer._detokenize_delta(
tokenizer, output_ids, DerenderStreamState()
)
streamed = self._chunked(derenderer, tokenizer, [[t] for t in output_ids])
engine = _decode_stream_expected(tokenizer, [], output_ids)
assert "�" not in batch
assert batch == streamed == engine
full, _ = derenderer._detokenize_delta(
tokenizer, byte_ids, DerenderStreamState()
)
assert full.endswith("🌈")
@pytest.mark.parametrize(
("skip", "spaces", "preserve", "fast", "expected"),
[
(True, True, False, True, (True, True)),
(True, False, False, True, (True, True)),
(True, False, False, False, (True, False)),
(False, False, False, True, (False, False)),
(False, False, True, True, (False, False)),
(True, False, True, True, (False, False)),
(True, True, True, True, (False, True)),
],
)
def test_decode_params_mirror_engine(
self, monkeypatch, skip, spaces, preserve, fast, expected
):
"""Flags match the engine's fast or slow detokenizer for the tokenizer."""
monkeypatch.setattr(
online_derenderer_module, "uses_fast_detokenizer", lambda _: fast
)
request = CompletionRequest(
model=MODEL_NAME,
prompt="x",
skip_special_tokens=skip,
spaces_between_special_tokens=spaces,
)
assert _decode_params(None, request, preserve_special=preserve) == expected
def test_decode_params_default_without_request(self, tokenizer):
assert _decode_params(tokenizer, None) == (True, True)
def test_n_independent_streams_same_result(self, derenderer, tokenizer):
"""N parallel streams with the same token sequence give the same text."""
token_ids = tokenizer.encode("parallel streams")[:8]
@@ -394,6 +516,242 @@ class TestDetokenizeDelta:
assert results[0] == self._one_shot(tokenizer, token_ids)
# ---------------------------------------------------------------------------
# Prompt seeded detokenization (matches the engine's primed detokenizer)
# ---------------------------------------------------------------------------
def _decode_stream_expected(
tokenizer, prompt_ids: list[int], output_ids: list[int]
) -> str:
"""Engine-equivalent decode via `DecodeStream`, primed by stepping the
prompt (`DecodeStream(ids=...)` needs tokenizers >= 0.22)."""
stream = DecodeStream(skip_special_tokens=True)
for tid in prompt_ids:
stream.step(tokenizer.backend_tokenizer, tid)
return "".join(
s
for tid in output_ids
if (s := stream.step(tokenizer.backend_tokenizer, tid)) is not None
)
@pytest.fixture(scope="module")
def leading_space_ids(tokenizer):
"""(prompt_ids, output_ids, expected) where seeded and unseeded decode
differ by the leading space."""
prompt_ids = tokenizer.encode("[INST] Hi [/INST]", add_special_tokens=False)
output_ids = tokenizer.encode("Hello there, output", add_special_tokens=False)
expected = _decode_stream_expected(tokenizer, prompt_ids, output_ids)
unseeded = tokenizer.decode(output_ids, skip_special_tokens=True)
assert expected.startswith(" ") and not unseeded.startswith(" "), (
"fixture no longer reproduces the leading-space divergence; "
"pick a different prompt/output pair"
)
return prompt_ids, output_ids, expected
class TestSeedStreamState:
"""`_seed_stream_state`: pure helper building the initial decode state."""
def test_seeds_prev_tokens_from_prompt_tail(self, tokenizer, leading_space_ids):
prompt_ids, _, _ = leading_space_ids
state = _seed_stream_state(tokenizer, prompt_ids, skip_special_tokens=True)
assert state.prev_tokens
assert state.read_offset == len(state.prev_tokens)
def test_none_prompt_token_ids_returns_empty_state(self, tokenizer):
assert (
_seed_stream_state(tokenizer, None, skip_special_tokens=True)
== DerenderStreamState()
)
def test_empty_prompt_token_ids_returns_empty_state(self, tokenizer):
assert (
_seed_stream_state(tokenizer, [], skip_special_tokens=True)
== DerenderStreamState()
)
class TestPromptSeededLeadingSpace:
"""Every decode entry point keeps the first token's leading space when
given prompt_token_ids."""
@pytest.mark.asyncio
async def test_chat_batch_seeded_via_explicit_prompt_token_ids(
self, derenderer, leading_space_ids
):
prompt_ids, output_ids, expected = leading_space_ids
choices = await derenderer.derender_chat(
GenerateResponse(
request_id="t",
choices=[
GenerateResponseChoice(
index=0, token_ids=output_ids, finish_reason="stop"
)
],
),
prompt_token_ids=prompt_ids,
)
assert choices[0].message.content == expected
@pytest.mark.asyncio
async def test_chat_batch_seeded_via_generate_response_fallback(
self, derenderer, leading_space_ids
):
prompt_ids, output_ids, expected = leading_space_ids
choices = await derenderer.derender_chat(
GenerateResponse(
request_id="t",
choices=[
GenerateResponseChoice(
index=0, token_ids=output_ids, finish_reason="stop"
)
],
prompt_token_ids=prompt_ids,
),
)
assert choices[0].message.content == expected
@pytest.mark.asyncio
async def test_chat_batch_explicit_takes_precedence_over_fallback(
self, derenderer, tokenizer, leading_space_ids
):
prompt_ids, output_ids, expected = leading_space_ids
wrong_ids = tokenizer.encode("a different prompt", add_special_tokens=False)
choices = await derenderer.derender_chat(
GenerateResponse(
request_id="t",
choices=[
GenerateResponseChoice(
index=0, token_ids=output_ids, finish_reason="stop"
)
],
prompt_token_ids=wrong_ids,
),
prompt_token_ids=prompt_ids,
)
assert choices[0].message.content == expected
@pytest.mark.asyncio
async def test_chat_batch_without_prompt_context_stays_unseeded(
self, derenderer, leading_space_ids
):
_, output_ids, _ = leading_space_ids
choices = await derenderer.derender_chat(
GenerateResponse(
request_id="t",
choices=[
GenerateResponseChoice(
index=0, token_ids=output_ids, finish_reason="stop"
)
],
),
)
assert not choices[0].message.content.startswith(" ")
@pytest.mark.asyncio
async def test_completion_batch_seeded(self, derenderer, leading_space_ids):
prompt_ids, output_ids, expected = leading_space_ids
choices, _, _ = await derenderer.derender_completion(
[
GenerateResponse(
request_id="t",
choices=[
GenerateResponseChoice(
index=0, token_ids=output_ids, finish_reason="stop"
)
],
)
],
prompt_token_ids=[prompt_ids],
)
assert choices[0].text == expected
@pytest.mark.asyncio
@pytest.mark.parametrize("from_generate_chunk", [False, True])
async def test_chat_stream_first_chunk_seeded(
self, derenderer, leading_space_ids, from_generate_chunk
):
prompt_ids, output_ids, expected = leading_space_ids
mid = len(output_ids) // 2
chunk1, state = await derenderer.derender_chat_stream(
model=MODEL_NAME,
generate_chunk=_make_stream_chunk(
output_ids[:mid],
prompt_token_ids=prompt_ids if from_generate_chunk else None,
),
prompt_token_ids=None if from_generate_chunk else prompt_ids,
)
# Only needed on the first chunk; the carried state already holds
# the seeded window from here on.
chunk2, _ = await derenderer.derender_chat_stream(
model=MODEL_NAME,
generate_chunk=_make_stream_chunk(output_ids[mid:], finish_reason="stop"),
state=state,
)
streamed = (chunk1.choices[0].delta.content or "") + (
chunk2.choices[0].delta.content or ""
)
assert streamed == expected
@pytest.mark.asyncio
@pytest.mark.parametrize("from_generate_chunk", [False, True])
async def test_completion_stream_first_chunk_seeded(
self, derenderer, leading_space_ids, from_generate_chunk
):
prompt_ids, output_ids, expected = leading_space_ids
mid = len(output_ids) // 2
chunk1, state = await derenderer.derender_completion_stream(
model=MODEL_NAME,
generate_chunk=_make_stream_chunk(
output_ids[:mid],
prompt_token_ids=prompt_ids if from_generate_chunk else None,
),
prompt_token_ids=None if from_generate_chunk else prompt_ids,
)
chunk2, _ = await derenderer.derender_completion_stream(
model=MODEL_NAME,
generate_chunk=_make_stream_chunk(output_ids[mid:], finish_reason="stop"),
state=state,
)
assert chunk1.choices[0].text + chunk2.choices[0].text == expected
@pytest.mark.asyncio
async def test_parser_path_ephemeral_state_seeded(
self, parsed_derenderer, monkeypatch, leading_space_ids
):
"""The parser path's per-call detok window is seeded too, so
`parse_delta` sees the same text as coupled serving."""
class _EchoParser(Parser):
def parse_delta(
self,
delta_text,
delta_token_ids,
request,
prompt_token_ids=None,
*,
finished,
):
return DeltaMessage(content=delta_text) if delta_text else None
monkeypatch.setattr(parsed_derenderer, "parser", _EchoParser)
prompt_ids, output_ids, expected = leading_space_ids
chunk, _ = await parsed_derenderer.derender_chat_stream(
model=MODEL_NAME,
generate_chunk=_make_stream_chunk(output_ids, finish_reason="stop"),
chat_request=_chat_request(),
prompt_token_ids=prompt_ids,
)
assert chunk.choices[0].delta.content == expected
@pytest.mark.asyncio
@pytest.mark.parametrize(
"stream_method",
@@ -403,8 +761,6 @@ async def test_streaming_detokenization_runs_off_event_loop(
derenderer, monkeypatch, stream_method
):
"""Streaming detokenization runs on the renderer executor."""
import vllm.renderers.online_derenderer as online_derenderer_module
event_loop_thread_id = threading.get_ident()
executor_thread_id = derenderer.renderer._executor.submit(
threading.get_ident
@@ -504,8 +860,6 @@ class TestDerenderCompletionStream:
@pytest.mark.asyncio
async def test_skip_special_tokens_threaded(self, derenderer, tokenizer):
"""completion_request.skip_special_tokens is honored (not hardcoded True)."""
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
eos = tokenizer.eos_token_id
if eos is None:
pytest.skip("tokenizer has no eos token to exercise special stripping")
@@ -536,6 +890,115 @@ class TestDerenderCompletionStream:
assert chunk.choices[0].finish_reason == "length"
class TestStreamLogprobs:
"""Streaming derender must carry per-chunk logprobs with placeholders
resolved, matching what the generate streaming path emits."""
@pytest.mark.asyncio
async def test_chat_stream_logprobs_resolved_per_chunk(self, derenderer, tokenizer):
"""Each streamed chunk carries logprobs with token_id:N resolved."""
token_ids = tokenizer.encode("hello world")[:6]
mid = len(token_ids) // 2
state = None
for part in (token_ids[:mid], token_ids[mid:]):
chunk, state = await derenderer.derender_chat_stream(
model=MODEL_NAME,
generate_chunk=_make_stream_chunk(
part, logprobs=_placeholder_logprobs(part)
),
state=state,
)
logprobs = chunk.choices[0].logprobs
assert logprobs is not None and logprobs.content is not None
assert len(logprobs.content) == len(part)
for entry in logprobs.content:
assert not entry.token.startswith("token_id:"), (
f"placeholder not resolved: {entry.token!r}"
)
@pytest.mark.asyncio
async def test_chat_stream_logprobs_multibyte_across_chunks(
self, derenderer, tokenizer
):
"""Byte-fallback correction works when a multi-byte character's
tokens are split across chunks (context carried in stream_state)."""
token_ids = tokenizer.encode("👍", add_special_tokens=False)
if len(token_ids) < 2 or "�" not in tokenizer.decode([token_ids[-1]]):
pytest.skip("Tokenizer does not byte-split this character")
state = None
last_chunk = None
# One token per chunk: every entry after the first needs cross-chunk
# context to resolve without U+FFFD.
for i, tid in enumerate(token_ids):
last_chunk, state = await derenderer.derender_chat_stream(
model=MODEL_NAME,
generate_chunk=_make_stream_chunk(
[tid], logprobs=_placeholder_logprobs([tid])
),
state=state,
)
assert last_chunk is not None
final_entry = last_chunk.choices[0].logprobs.content[0]
assert not final_entry.token.endswith("�"), (
"byte-fallback correction failed across chunk boundary: "
f"{final_entry.token!r}"
)
@pytest.mark.asyncio
async def test_completion_stream_logprobs_text_offset_absolute(
self, derenderer, tokenizer
):
"""text_offset continues across chunks instead of restarting at 0."""
token_ids = tokenizer.encode("hello world")[:6]
mid = len(token_ids) // 2
chunk1, state = await derenderer.derender_completion_stream(
model=MODEL_NAME,
generate_chunk=_make_stream_chunk(
token_ids[:mid], logprobs=_placeholder_logprobs(token_ids[:mid])
),
)
chunk2, _ = await derenderer.derender_completion_stream(
model=MODEL_NAME,
generate_chunk=_make_stream_chunk(
token_ids[mid:], logprobs=_placeholder_logprobs(token_ids[mid:])
),
state=state,
)
assert chunk1.choices[0].logprobs.text_offset[0] == 0
assert chunk2.choices[0].logprobs.text_offset[0] == len(chunk1.choices[0].text)
@pytest.mark.asyncio
async def test_stream_logprob_state_stays_bounded(self, derenderer, tokenizer):
"""Carried logprob context never exceeds the 4-token window."""
token_ids = tokenizer.encode("the quick brown fox jumps over the lazy dog")
state = None
for tid in token_ids:
_, state = await derenderer.derender_chat_stream(
model=MODEL_NAME,
generate_chunk=_make_stream_chunk(
[tid], logprobs=_placeholder_logprobs([tid])
),
state=state,
)
assert state is not None
assert len(state.logprob_context_token_ids) <= 4
@pytest.mark.asyncio
async def test_stream_without_logprobs_unchanged(self, derenderer, tokenizer):
"""Chunks without logprobs keep logprobs=None on the output choice."""
token_ids = tokenizer.encode("hello")[:3]
chunk, _ = await derenderer.derender_chat_stream(
model=MODEL_NAME,
generate_chunk=_make_stream_chunk(token_ids),
)
assert chunk.choices[0].logprobs is None
class TestDerenderChatStream:
"""derender_chat_stream: plain detok branch (no parser)."""
@@ -623,8 +1086,6 @@ class TestDerenderChatStream:
def _chat_request(**kwargs):
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
kwargs.setdefault("messages", [{"role": "user", "content": "hi"}])
kwargs.setdefault("model", MODEL_NAME)
return ChatCompletionRequest(**kwargs)
@@ -906,8 +1367,6 @@ def harmony_encode():
@pytest.fixture(scope="module")
def harmony_tokenizer():
pytest.importorskip("openai_harmony")
from vllm.tokenizers import get_tokenizer
return get_tokenizer(HARMONY_MODEL, trust_remote_code=True)
@@ -919,10 +1378,7 @@ def harmony_derenderer(harmony_tokenizer):
with a real tokenizer and parser, so the replay path is exercised
against Harmony's actual channel grammar rather than a stub.
"""
from unittest.mock import MagicMock
from vllm.parser.harmony import HarmonyParser
from vllm.renderers.online_derenderer import OnlineDerenderer
renderer = MagicMock()
renderer.get_tokenizer.return_value = harmony_tokenizer
@@ -1058,20 +1514,14 @@ class TestDerenderStreamStateValidation:
"""DerenderStreamState rejects malformed caller supplied offsets/lengths."""
def test_negative_prefix_offset_rejected(self):
from pydantic import ValidationError
with pytest.raises(ValidationError):
DerenderStreamState(prefix_offset=-1)
def test_negative_read_offset_rejected(self):
from pydantic import ValidationError
with pytest.raises(ValidationError):
DerenderStreamState(read_offset=-1)
def test_prev_tokens_over_cap_rejected(self):
from pydantic import ValidationError
with pytest.raises(ValidationError):
DerenderStreamState(prev_tokens=["a"] * 1025)
@@ -1080,14 +1530,10 @@ class TestDerenderStreamStateValidation:
assert len(state.prev_tokens) == 1024
def test_output_chunk_lens_mismatch_rejected(self):
from pydantic import ValidationError
with pytest.raises(ValidationError):
DerenderStreamState(output_token_ids=[1, 2, 3], output_chunk_lens=[1, 1])
def test_output_chunk_lens_zero_entry_rejected(self):
from pydantic import ValidationError
with pytest.raises(ValidationError):
DerenderStreamState(output_token_ids=[1, 2], output_chunk_lens=[0, 2])
@@ -1098,14 +1544,40 @@ class TestDerenderStreamStateValidation:
assert state.output_chunk_lens == [1, 2]
_GEN = {"request_id": "t", "choices": []}
_GEN_BAD = {**_GEN, "prompt_token_ids": [-1]}
@pytest.mark.parametrize(
"model_cls,payload",
[
(DerenderChatRequest, {"generate_response": _GEN, "prompt_token_ids": [-1]}),
(DerenderChatRequest, {"generate_response": _GEN_BAD}),
(
DerenderCompletionRequest,
{"generate_responses": [_GEN], "prompt_token_ids": [[-1]]},
),
(
DerenderChatStreamRequest,
{"stream": True, "generate_chunk": _GEN, "prompt_token_ids": [-1]},
),
(
DerenderCompletionStreamRequest,
{"stream": True, "generate_chunk": _GEN_BAD},
),
],
)
def test_negative_prompt_token_ids_rejected(model_cls, payload):
"""Negative prompt ids would reach convert_ids_to_tokens, so they get a
400 at parse time like `GenerateRequest.token_ids` does."""
with pytest.raises(ValidationError, match="prompt_token_ids"):
model_cls.model_validate(payload)
class TestServingDerenderStreamErrorHandling:
"""Malformed stream_state must surface as 400 and not an unhandled 500."""
def _make_serving(self, side_effect: Exception):
from unittest.mock import AsyncMock, MagicMock
from vllm.entrypoints.scale_out.derender.serving import ServingDerender
models = MagicMock()
models.is_base_model.return_value = True
models.model_config = MagicMock()
@@ -1123,10 +1595,6 @@ class TestServingDerenderStreamErrorHandling:
@pytest.mark.asyncio
@pytest.mark.parametrize("exc", [KeyError("bad byte"), IndexError("oob")])
async def test_completion_stream_bad_state_returns_400(self, exc):
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
DerenderCompletionStreamRequest,
)
serving = self._make_serving(exc)
request = DerenderCompletionStreamRequest(
stream=True,
@@ -1141,10 +1609,6 @@ class TestServingDerenderStreamErrorHandling:
@pytest.mark.asyncio
@pytest.mark.parametrize("exc", [KeyError("bad byte"), IndexError("oob")])
async def test_chat_stream_bad_state_returns_400(self, exc):
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
DerenderChatStreamRequest,
)
serving = self._make_serving(exc)
request = DerenderChatStreamRequest(
stream=True,
@@ -1163,13 +1627,6 @@ class TestServingDerenderStreamValidation:
before touching the tokenizer)."""
def _make_serving(self, *, parser_configured: bool, max_model_len: int = 100_000):
from unittest.mock import AsyncMock, MagicMock
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionStreamResponse,
)
from vllm.entrypoints.scale_out.derender.serving import ServingDerender
models = MagicMock()
models.is_base_model.return_value = True
models.model_config = MagicMock()
@@ -1187,10 +1644,6 @@ class TestServingDerenderStreamValidation:
@pytest.mark.asyncio
async def test_missing_chat_request_with_parser_rejected(self):
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
DerenderChatStreamRequest,
)
serving = self._make_serving(parser_configured=True)
request = DerenderChatStreamRequest(
stream=True,
@@ -1205,10 +1658,6 @@ class TestServingDerenderStreamValidation:
@pytest.mark.asyncio
async def test_missing_chat_request_without_parser_ok(self):
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
DerenderChatStreamRequest,
)
serving = self._make_serving(parser_configured=False)
request = DerenderChatStreamRequest(
stream=True,
@@ -1224,10 +1673,6 @@ class TestServingDerenderStreamValidation:
the same way it rejects a missing chat_request. Without it,
parse_delta cannot tell whether the prompt left reasoning open and
would misclassify reasoning content as plain content."""
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
DerenderChatStreamRequest,
)
serving = self._make_serving(parser_configured=True)
request = DerenderChatStreamRequest(
stream=True,
@@ -1243,10 +1688,6 @@ class TestServingDerenderStreamValidation:
@pytest.mark.asyncio
async def test_prompt_token_ids_present_with_parser_ok(self):
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
DerenderChatStreamRequest,
)
serving = self._make_serving(parser_configured=True)
request = DerenderChatStreamRequest(
stream=True,
@@ -1260,10 +1701,6 @@ class TestServingDerenderStreamValidation:
@pytest.mark.asyncio
async def test_oversized_output_token_ids_rejected(self):
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
DerenderChatStreamRequest,
)
serving = self._make_serving(parser_configured=False, max_model_len=4)
request = DerenderChatStreamRequest(
stream=True,
@@ -1285,10 +1722,6 @@ class TestServingDerenderStreamValidation:
parser configured deployment rescans it in full on every chunk
(is_reasoning_end / adjust_initial_state_from_prompt), so it must be
bounded the same way output_token_ids is."""
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
DerenderChatStreamRequest,
)
serving = self._make_serving(parser_configured=True, max_model_len=4)
request = DerenderChatStreamRequest(
stream=True,
@@ -1309,10 +1742,6 @@ class TestServingDerenderStreamValidation:
DerenderStreamState is threaded through every choice). The check
could never fire since derender_chat_stream itself rejects anything
above 1 first."""
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
DerenderChatStreamRequest,
)
serving = self._make_serving(parser_configured=False)
two_choices = GenerateStreamResponse(
request_id="t",
@@ -1337,16 +1766,12 @@ class TestServingDerenderStreamValidation:
@pytest.fixture(scope="module")
def server():
from tests.utils import RemoteLaunchRenderServer
with RemoteLaunchRenderServer(MODEL_NAME, []) as remote_server:
yield remote_server
@pytest_asyncio.fixture
async def client(server):
import httpx
async with httpx.AsyncClient(
base_url=server.url_for(""), timeout=30.0
) as http_client:
@@ -1610,8 +2035,6 @@ _PARSER_TOOLS = [
@pytest.fixture(scope="module")
def parser_server():
from tests.utils import RemoteLaunchRenderServer
args = [
"--enable-auto-tool-choice",
"--tool-call-parser",
@@ -1625,8 +2048,6 @@ def parser_server():
@pytest_asyncio.fixture
async def parser_client(parser_server):
import httpx
async with httpx.AsyncClient(
base_url=parser_server.url_for(""), timeout=60.0
) as http_client:
@@ -1635,8 +2056,6 @@ async def parser_client(parser_server):
@pytest.fixture(scope="module")
def parser_tokenizer():
from vllm.tokenizers import get_tokenizer
return get_tokenizer(PARSER_MODEL)
@@ -15,15 +15,26 @@ from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.scale_out.token_in_token_out.mm_features import (
placeholder_ranges_from_engine_input,
)
from vllm.entrypoints.scale_out.token_in_token_out.mm_serde import (
encode_mm_kwargs_item,
)
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
GenerateRequest,
GenerateResponse,
MultiModalFeatures,
PlaceholderRangeInfo,
)
from vllm.entrypoints.scale_out.token_in_token_out.serving import ServingTokens
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.exceptions import GenerationError
from vllm.logprobs import Logprob
from vllm.multimodal.inputs import PlaceholderRange
from vllm.outputs import CompletionOutput, RequestOutput
from vllm.multimodal.inputs import (
MultiModalBatchedField,
MultiModalFieldElem,
MultiModalKwargsItem,
PlaceholderRange,
)
from vllm.outputs import CompletionOutput, RequestOutput, SamplingMask
from vllm.renderers import renderer_from_config
from vllm.renderers.online_renderer import OnlineRenderer
from vllm.sampling_params import SamplingParams
@@ -285,6 +296,46 @@ async def test_non_stream_error():
await serving.serve_tokens(request)
@pytest.mark.asyncio
async def test_serve_tokens_rejects_invalid_shm_handle_before_engine_handoff():
engine = _mock_engine()
engine.generate = MagicMock()
serving = _build_serving_tokens(engine)
cache = MagicMock()
cache.validate_input_item.side_effect = ValueError(
"Invalid SHM handle signature for cache key."
)
serving.online_renderer.renderer._mm_processor_cache = cache
forged_item = MultiModalKwargsItem(
{
"address": MultiModalFieldElem(data=224, field=MultiModalBatchedField()),
"monotonic_id": MultiModalFieldElem(
data=4242,
field=MultiModalBatchedField(),
),
}
)
request = GenerateRequest(
token_ids=[1, 2, 3],
sampling_params=SamplingParams(max_tokens=1),
model=MODEL_NAME,
features=MultiModalFeatures(
mm_hashes={"image": ["image_A"]},
mm_placeholders={"image": [PlaceholderRangeInfo(offset=0, length=1)]},
kwargs_data={"image": [encode_mm_kwargs_item(forged_item)]},
),
)
response = await serving.serve_tokens(request)
assert isinstance(response, ErrorResponse)
cache.validate_input_item.assert_called_once()
validated_item, validated_hash = cache.validate_input_item.call_args.args
assert validated_hash == "image_A"
assert validated_item["address"].data == 224
engine.generate.assert_not_called()
@pytest.mark.asyncio
async def test_stream_basic():
"""Streaming returns SSE chunks with correct token_ids and ends with [DONE]."""
@@ -1103,3 +1154,44 @@ async def test_stream_prompt_tokens_details_zero_cached():
# Zero cached tokens must be present, not omitted
assert usage_chunk["usage"]["prompt_tokens_details"] is not None
assert usage_chunk["usage"]["prompt_tokens_details"]["cached_tokens"] == 0
@pytest.mark.asyncio
@pytest.mark.parametrize("with_mask", [False, True])
async def test_stream_sampling_mask_matches_each_token_chunk(with_mask):
engine = _mock_engine()
async def generate(*args, **kwargs):
for position, tokens in enumerate(([10], [20, 30])):
result = _make_request_output(
"req-mask",
token_ids=list(tokens),
finish_reason="length" if position else None,
finished=bool(position),
)
if with_mask:
result.outputs[0].sampling_mask = SamplingMask(
[[token, token + 1] for token in tokens]
)
yield result
engine.generate = MagicMock(side_effect=generate)
serving = _build_serving_tokens(engine)
response = await serving.serve_tokens(
GenerateRequest(
model=MODEL_NAME, token_ids=[1, 2, 3], sampling_params={}, stream=True
)
)
chunks = _parse_sse_chunks([chunk async for chunk in response])
choices = [
choice
for chunk in chunks
if isinstance(chunk, dict)
for choice in chunk.get("choices", [])
]
assert [choice["token_ids"] for choice in choices] == [[10], [20, 30]]
for choice in choices:
expected = (
[[token, token + 1] for token in choice["token_ids"]] if with_mask else None
)
assert choice["sampling_mask"] == expected
@@ -0,0 +1,139 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
from unittest.mock import patch
import pytest
import regex as re
import requests
import torch
from transformers import AutoModelForCausalLM
from tests.entrypoints.serve.dev.rlhf.conftest import (
MODEL_NAME,
gen,
health,
ok,
server,
)
from tests.utils import multi_gpu_test
from vllm.distributed.weight_transfer import (
HTTPVLLMWeightSyncClient,
ModuleSource,
WeightTransferTrainerFactory,
)
from vllm.distributed.weight_transfer.ipc_engine import IPCTrainerInitInfo
from vllm.platforms import current_platform
def weight_checker(url: str, action: str, baseline=None) -> requests.Response:
body = {"action": action}
if baseline is not None:
body["baseline"] = baseline
return requests.post(f"{url}/weight_checker", json=body, timeout=300)
def checksums(url: str) -> dict[str, str]:
response = weight_checker(url, "checksum")
response.raise_for_status()
return response.json()["checksums"]
def ranks(keys) -> set[str]:
return {key.rsplit(":", 1)[0] for key in keys}
def reset_reload_and_compare(url: str) -> None:
baseline = checksums(url)
try:
weight_checker(url, "reset").raise_for_status()
comparison = weight_checker(url, "compare", baseline).json()
assert ranks(comparison["mismatches"]) == ranks(baseline)
finally:
requests.post(
f"{url}/collective_rpc", json={"method": "reload_weights"}, timeout=300
).raise_for_status()
assert weight_checker(url, "compare", baseline).json() == {
"match": True,
"mismatches": [],
}
@pytest.fixture(scope="class")
def server_url():
with server() as url:
yield url
class TestSingleGPU:
@pytest.mark.parametrize(
"payload",
[{}, {"action": "frobnicate"}, {"action": "compare"}, [], None],
)
def test_invalid_request_returns_400(self, server_url, payload):
response = requests.post(
f"{server_url}/weight_checker", json=payload, timeout=10
)
assert response.status_code == 400, response.text
assert health(server_url) == 200
def test_checksum_is_stable_and_compare_reports_differences(self, server_url):
baseline = checksums(server_url)
assert all(
re.fullmatch(r"[0-9a-f]{64}", digest) for digest in baseline.values()
)
assert checksums(server_url) == baseline
changed, missing, *_ = baseline
altered = {**baseline, changed: "0" * 64}
del altered[missing]
assert weight_checker(server_url, "compare", altered).json() == {
"match": False,
"mismatches": sorted([changed, missing]),
}
def test_reset_changes_weights_and_reload_restores_them(self, server_url):
reset_reload_and_compare(server_url)
assert ok(gen(server_url))
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="IPC weight transfer uses CUDA IPC."
)
def test_ipc_weight_transfer_restores_reset_weights():
"""The documented RL flow: reset, then transfer the checkpoint back."""
if current_platform.is_xpu():
pytest.skip("IPC weight transfer backend uses CUDA IPC handles")
args = ["--weight-transfer-config", '{"backend": "ipc"}']
with (
patch.dict(os.environ, {"VLLM_ALLOW_INSECURE_SERIALIZATION": "1"}),
server(extra_args=args, port=8772) as url,
):
baseline = checksums(url)
weight_checker(url, "reset").raise_for_status()
trainer_model = AutoModelForCausalLM.from_pretrained(
MODEL_NAME, torch_dtype=torch.bfloat16
).cuda()
WeightTransferTrainerFactory.trainer_init(
IPCTrainerInitInfo(rank=0),
client=HTTPVLLMWeightSyncClient(url),
source=ModuleSource(trainer_model),
).send_weights()
assert weight_checker(url, "compare", baseline).json() == {
"match": True,
"mismatches": [],
}
@multi_gpu_test(num_gpus=2)
@pytest.mark.parametrize(
"args, expected",
[
(["--data-parallel-size", "2"], {"dp0:pp0:pcp0:tp0", "dp1:pp0:pcp0:tp0"}),
(["--tensor-parallel-size", "2"], {"dp0:pp0:pcp0:tp0", "dp0:pp0:pcp0:tp1"}),
],
)
def test_checksum_and_reset_cover_every_worker(args, expected):
with server(extra_args=args, port=8771, timeout=600) as url:
assert ranks(checksums(url)) == expected
reset_reload_and_compare(url)
@@ -0,0 +1,93 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Weight-transfer metrics scraped from a real two-API-server deployment."""
from pathlib import Path
import requests
import torch
from prometheus_client.multiprocess import MultiProcessCollector
from prometheus_client.parser import text_string_to_metric_families
from transformers import AutoModelForCausalLM
from tests.entrypoints.serve.dev.rlhf.conftest import MODEL_NAME
from tests.utils import RemoteOpenAIServer
from vllm.distributed.weight_transfer import (
HTTPVLLMWeightSyncClient,
ModuleSource,
WeightTransferTrainerFactory,
)
from vllm.distributed.weight_transfer.ipc_engine import IPCTrainerInitInfo
DURATION = "vllm:rl_weight_update_operation_duration_seconds"
IN_FLIGHT = "vllm:rl_weight_update_operations_in_flight"
# The API servers accept from one shared listening socket, so the kernel may
# hand one process most connections. Sync until both have recorded, within a cap.
MIN_SYNCS = 8
MAX_SYNCS = 64
def scrape(url: str) -> dict[tuple[str, str], float]:
text = requests.get(f"{url}/metrics", timeout=30).text
return {
(sample.name, sample.labels["operation"]): sample.value
for family in text_string_to_metric_families(text)
for sample in family.samples
if sample.name in (f"{DURATION}_count", IN_FLIGHT)
}
def pids_that_recorded(multiproc_dir: Path) -> set[str]:
return {
path.stem.rsplit("_", 1)[1]
for path in multiproc_dir.glob("histogram_*.db")
if any(
metric.name == DURATION
for metric in MultiProcessCollector.merge([str(path)])
)
}
def test_weight_sync_metrics_aggregate_across_api_servers(tmp_path):
args = [
"--load-format",
"dummy",
"--enforce-eager",
"--max-model-len",
"512",
"--gpu-memory-utilization",
"0.5",
"--api-server-count",
"2",
"--weight-transfer-config",
'{"backend": "ipc"}',
]
env = {
"VLLM_SERVER_DEV_MODE": "1",
"VLLM_ALLOW_INSECURE_SERIALIZATION": "1",
"PROMETHEUS_MULTIPROC_DIR": str(tmp_path),
}
with RemoteOpenAIServer(MODEL_NAME, args, env_dict=env) as server:
url = server.url_root
trainer = AutoModelForCausalLM.from_pretrained(
MODEL_NAME, torch_dtype=torch.bfloat16
).cuda()
engine = WeightTransferTrainerFactory.trainer_init(
IPCTrainerInitInfo(rank=0),
client=HTTPVLLMWeightSyncClient(url),
source=ModuleSource(trainer),
)
syncs = 0
while syncs < MIN_SYNCS or (
len(pids_that_recorded(tmp_path)) < 2 and syncs < MAX_SYNCS
):
engine.send_weights()
syncs += 1
metrics = scrape(url)
calls = {"init": 1, "start": syncs, "update": syncs, "finish": syncs}
for operation, count in calls.items():
assert metrics[(f"{DURATION}_count", operation)] == count, operation
assert metrics[(IN_FLIGHT, operation)] == 0, operation
# The counts above are sums over processes only if both servers recorded.
assert len(pids_that_recorded(tmp_path)) == 2
+179 -5
View File
@@ -1,35 +1,209 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
from types import SimpleNamespace
from unittest.mock import AsyncMock
import pytest
import requests
from fastapi import FastAPI
from fastapi.testclient import TestClient
from prometheus_client import CollectorRegistry
from prometheus_client.parser import text_string_to_metric_families
from tests.utils import RemoteOpenAIServer
from vllm.entrypoints.serve.dev.sleep import metrics as sleep_metrics
from vllm.entrypoints.serve.dev.sleep.api_router import attach_router
from vllm.entrypoints.serve.dev.sleep.metrics import SleepModeOperationMetrics
from vllm.entrypoints.serve.exception_handling.register import init_exception_handler
from vllm.entrypoints.serve.instrumentator.metrics import (
attach_router as attach_metrics_router,
)
from vllm.v1.metrics import prometheus as prometheus_metrics
MODEL_NAME = "meta-llama/Llama-3.2-1B"
@pytest.fixture
def sleep_route_app(monkeypatch):
app = FastAPI()
app.state.args = SimpleNamespace(log_error_stack=False)
app.state.engine_client = AsyncMock()
metrics = SleepModeOperationMetrics(CollectorRegistry())
monkeypatch.setattr(
"vllm.entrypoints.serve.dev.sleep.api_router.sleep_mode_operation_metrics",
lambda: metrics,
)
attach_router(app)
init_exception_handler(app)
return app, metrics
@pytest.mark.cpu_test
@pytest.mark.parametrize("level", [0, 1, 2])
def test_sleep_route_response_and_engine_arguments(sleep_route_app, level):
app, _ = sleep_route_app
with TestClient(app) as client:
response = client.post("/sleep", params={"level": level})
assert response.status_code == 200
assert response.json() == {"status": "sleeping", "level": level}
app.state.engine_client.sleep.assert_awaited_once_with(level, "abort")
@pytest.mark.cpu_test
@pytest.mark.parametrize(
("query", "expected_param"),
[
("level=invalid", "query.level"),
("level=-1", "query.level"),
("level=3", "query.level"),
("mode=invalid", "query.mode"),
],
)
def test_sleep_route_rejects_invalid_query_before_dispatch(
sleep_route_app, query, expected_param
):
app, metrics = sleep_route_app
with TestClient(app) as client:
response = client.post(f"/sleep?{query}")
assert response.status_code == 400
assert response.json()["error"]["param"] == expected_param
app.state.engine_client.sleep.assert_not_awaited()
assert list(metrics.duration.collect()[0].samples) == []
assert list(metrics.in_flight.collect()[0].samples) == []
@pytest.mark.cpu_test
@pytest.mark.parametrize(
("fully_awake", "tags", "expected"),
[
(True, "", {"status": "awake", "tags": None}),
(False, "?tags=weights", {"status": "sleeping", "tags": ["weights"]}),
],
)
def test_wake_route_maps_engine_result_without_state_query(
sleep_route_app, fully_awake, tags, expected
):
app, _ = sleep_route_app
app.state.engine_client.wake_up.return_value = fully_awake
with TestClient(app) as client:
response = client.post(f"/wake_up{tags}")
assert response.status_code == 200
assert response.json() == expected
app.state.engine_client.wake_up.assert_awaited_once_with(expected["tags"])
app.state.engine_client.is_sleeping.assert_not_awaited()
@pytest.mark.cpu_test
@pytest.mark.parametrize("fails", [False, True])
def test_release_kv_cache_memory_route(fails):
app = FastAPI()
app.state.engine_client = AsyncMock()
def test_release_kv_cache_memory_route(sleep_route_app, fails):
app, metrics = sleep_route_app
release = app.state.engine_client.release_kv_cache_memory
if fails:
release.side_effect = RuntimeError("requires a completed pause first")
attach_router(app)
with TestClient(app, raise_server_exceptions=False) as client:
response = client.post("/release_kv_cache_memory")
assert response.status_code == (500 if fails else 200)
if not fails:
assert response.json() == {"status": "kv_cache_released"}
release.assert_awaited_once_with()
assert metrics.in_flight.labels("release_kv_cache_memory")._value.get() == 0
assert _duration_count(metrics, "release_kv_cache_memory") == 1
@pytest.mark.cpu_test
@pytest.mark.parametrize("operation", ["sleep", "release_kv_cache_memory", "wake"])
@pytest.mark.parametrize("exception", [RuntimeError, asyncio.CancelledError])
def test_sleep_mode_recorder_tracks_duration_and_cleans_up(operation, exception):
metrics = SleepModeOperationMetrics(CollectorRegistry())
with metrics.record(operation):
assert metrics.in_flight.labels(operation)._value.get() == 1
with pytest.raises(exception), metrics.record(operation):
raise exception()
assert metrics.in_flight.labels(operation)._value.get() == 0
assert metrics.duration.labels(operation)._sum.get() >= 0
assert _duration_count(metrics, operation) == 2
def _duration_count(metrics, operation):
return next(
sample.value
for sample in metrics.duration.collect()[0].samples
if sample.name.endswith("_count") and sample.labels == {"operation": operation}
)
@pytest.mark.cpu_test
@pytest.mark.parametrize(
("path", "method", "operation", "status", "fails"),
[
("/sleep", "POST", "sleep", 200, False),
("/release_kv_cache_memory", "POST", "release_kv_cache_memory", 200, False),
("/wake_up", "POST", "wake", 200, False),
("/is_sleeping", "GET", None, 200, False),
("/sleep?level=3", "POST", None, 400, False),
("/release_kv_cache_memory", "POST", "release_kv_cache_memory", 500, True),
],
)
def test_sleep_routes_visible_on_production_metrics_endpoint(
monkeypatch, path, method, operation, status, fails
):
registry = CollectorRegistry()
monkeypatch.delenv("PROMETHEUS_MULTIPROC_DIR", raising=False)
monkeypatch.setattr(sleep_metrics, "REGISTRY", registry)
monkeypatch.setattr(sleep_metrics, "_metrics", None)
monkeypatch.setattr(prometheus_metrics, "REGISTRY", registry)
app = FastAPI()
app.state.args = SimpleNamespace(log_error_stack=False)
app.state.engine_client = AsyncMock()
app.state.engine_client.wake_up.return_value = True
app.state.engine_client.is_sleeping.return_value = False
if fails:
app.state.engine_client.release_kv_cache_memory.side_effect = RuntimeError(
"engine failed"
)
attach_router(app)
init_exception_handler(app)
attach_metrics_router(app)
with TestClient(app, raise_server_exceptions=False) as client:
assert client.request(method, path).status_code == status
response = client.get("/metrics")
assert response.status_code == 200
samples = [
sample
for family in text_string_to_metric_families(response.text)
for sample in family.samples
]
assert any(
sample.name == "http_requests_total"
and sample.labels
== {
"handler": path.split("?")[0],
"method": method,
"status": f"{status // 100}xx",
}
and sample.value == 1
for sample in samples
)
assert not any("sleep_mode_operations_total" in sample.name for sample in samples)
assert any(
sample.name == "vllm:sleep_mode_operations_in_flight"
and sample.labels == {"operation": operation}
and sample.value == 0
for sample in samples
) == (operation is not None)
assert any(
sample.name == "vllm:sleep_mode_operation_duration_seconds_count"
and sample.labels == {"operation": operation}
and sample.value == 1
for sample in samples
) == (operation is not None)
def test_sleep_mode():
@@ -29,7 +29,9 @@ LORA_UNLOADING_SUCCESS_MESSAGE = (
)
async def _async_serving_models_init() -> OpenAIServingModels:
async def _async_serving_models_init_with_mock() -> tuple[
OpenAIServingModels, MagicMock
]:
mock_engine_client = MagicMock(spec=EngineClient)
# Set the max_model_len attribute to avoid missing attribute
mock_model_config = MagicMock(spec=ModelConfig)
@@ -45,6 +47,13 @@ async def _async_serving_models_init() -> OpenAIServingModels:
)
await serving_models.init_static_loras()
# The mock is returned separately: `engine_client` is typed as the protocol,
# so assertion helpers are not visible through it.
return serving_models, mock_engine_client
async def _async_serving_models_init() -> OpenAIServingModels:
serving_models, _ = await _async_serving_models_init_with_mock()
return serving_models
@@ -99,6 +108,22 @@ async def test_load_lora_adapter_duplicate():
assert len(serving_models.lora_requests) == 1
@pytest.mark.asyncio
@pytest.mark.parametrize("load_inplace", [False, True])
async def test_load_lora_adapter_base_model_name(load_inplace: bool):
"""A LoRA named after a served model would shadow it for all requests."""
serving_models, mock_engine_client = await _async_serving_models_init_with_mock()
request = LoadLoRAAdapterRequest(
lora_name=MODEL_NAME, lora_path="/path/to/adapter", load_inplace=load_inplace
)
response = await serving_models.load_lora_adapter(request)
assert isinstance(response, ErrorResponse)
assert response.error.type == "InvalidUserInput"
assert response.error.code == HTTPStatus.BAD_REQUEST
assert len(serving_models.lora_requests) == 0
mock_engine_client.add_lora.assert_not_called()
@pytest.mark.asyncio
async def test_unload_lora_adapter_success():
serving_models = await _async_serving_models_init()
@@ -108,8 +133,8 @@ async def test_unload_lora_adapter_success():
response = await serving_models.load_lora_adapter(request)
assert len(serving_models.lora_requests) == 1
request = UnloadLoRAAdapterRequest(lora_name="adapter1")
response = await serving_models.unload_lora_adapter(request)
unload_request = UnloadLoRAAdapterRequest(lora_name="adapter1")
response = await serving_models.unload_lora_adapter(unload_request)
assert response == LORA_UNLOADING_SUCCESS_MESSAGE.format(lora_name="adapter1")
assert len(serving_models.lora_requests) == 0
@@ -84,7 +84,10 @@ def task_routes(request, monkeypatch) -> tuple[str, list[tuple[str, list[str]]]]
# Register routers for this specific task (development mode already enabled).
register_api_routers(
args, app, supported_tasks=(task,), model_config=MockModelConfig()
args,
app,
supported_tasks=(task,),
model_config=MockModelConfig(),
)
routes = get_all_http_routes(app)
@@ -19,6 +19,7 @@ from vllm.entrypoints.serve.tokenize.api_router import attach_router
from vllm.entrypoints.serve.tokenize.protocol import (
TokenizeChatRequest,
TokenizeCompletionRequest,
TokenizeResponse,
)
from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
from vllm.exceptions import VLLMNotFoundError, VLLMValidationError
@@ -95,7 +96,7 @@ async def test_tokenize_chat_skips_mm_cache_for_renderer_only_path():
mock_engine.renderer = MagicMock()
serving = _build_serving_tokenization(mock_engine)
serving.online_renderer.preprocess_chat = AsyncMock(
serving.online_renderer.preprocess_chat = AsyncMock( # type: ignore[method-assign]
return_value=(
[{"role": "user", "content": "Test"}],
[{"prompt_token_ids": [1, 2, 3]}],
@@ -109,7 +110,7 @@ async def test_tokenize_chat_skips_mm_cache_for_renderer_only_path():
response = await serving.create_tokenize(request, MagicMock(headers={}))
assert response.tokens == [1, 2, 3]
assert isinstance(response, TokenizeResponse) and response.tokens == [1, 2, 3]
assert (
serving.online_renderer.preprocess_chat.call_args.kwargs["skip_mm_cache"]
is True
@@ -125,7 +126,7 @@ async def test_tokenize_completion_skips_mm_cache_for_renderer_only_path():
mock_engine.renderer = MagicMock()
serving = _build_serving_tokenization(mock_engine)
serving.online_renderer.preprocess_completion = AsyncMock(
serving.online_renderer.preprocess_completion = AsyncMock( # type: ignore[method-assign]
return_value=[{"prompt_token_ids": [1, 2, 3]}]
)
@@ -136,7 +137,7 @@ async def test_tokenize_completion_skips_mm_cache_for_renderer_only_path():
response = await serving.create_tokenize(request, MagicMock(headers={}))
assert response.tokens == [1, 2, 3]
assert isinstance(response, TokenizeResponse) and response.tokens == [1, 2, 3]
assert (
serving.online_renderer.preprocess_completion.call_args.kwargs["skip_mm_cache"]
is True
@@ -101,7 +101,7 @@ async def test_tokenize_chat(
if continue_final:
conversation.append({"role": "assistant", "content": "Sure,"})
prompt = tokenizer.apply_chat_template(
prompt = tokenizer.apply_chat_template( # type: ignore[call-arg]
add_generation_prompt=add_generation,
continue_final_message=continue_final,
conversation=conversation,
@@ -169,7 +169,7 @@ async def test_tokenize_chat_with_tools(
if continue_final:
conversation.append({"role": "assistant", "content": "Sure,"})
prompt = tokenizer.apply_chat_template(
prompt = tokenizer.apply_chat_template( # type: ignore[call-arg]
add_generation_prompt=add_generation,
continue_final_message=continue_final,
conversation=conversation,
@@ -18,7 +18,7 @@ def _cfg(tp=1, pp=1, dp=1, ep=False, digest="a3b21f94deadbeef"):
enable_expert_parallel=ep,
)
)
c.compute_hash = lambda: digest # type: ignore[attr-defined]
c.compute_hash = lambda: digest
return c
@@ -59,6 +59,7 @@ def test_get_respects_set_default():
fp.set_default_fingerprint_mode("hash")
hashed = fp.get_system_fingerprint(cfg)
assert hashed is not None
assert hashed != full
assert "tp8" not in hashed
@@ -72,5 +73,7 @@ def test_get_respects_set_default():
def test_compute_hash_failure_does_not_raise():
cfg = _cfg()
cfg.compute_hash = lambda: (_ for _ in ()).throw(RuntimeError("boom"))
assert fp.build_system_fingerprint(cfg, "full").endswith("-nohash")
assert fp.build_system_fingerprint(cfg, "hash").endswith("-nohash")
full_fp = fp.build_system_fingerprint(cfg, "full")
assert full_fp is not None and full_fp.endswith("-nohash")
hash_fp = fp.build_system_fingerprint(cfg, "hash")
assert hash_fp is not None and hash_fp.endswith("-nohash")
@@ -47,7 +47,7 @@ def test_sanitize_handles_falsy_inputs() -> None:
assert _sanitize_transcription_user_text("") == ""
# The dataclass default for ``response_prefix`` is the empty string;
# the sanitizer must accept that without exception or extra work.
assert _sanitize_transcription_user_text(None) == "" # type: ignore[arg-type]
assert _sanitize_transcription_user_text(None) == ""
def test_sanitize_is_idempotent() -> None:
@@ -117,10 +117,7 @@ MESSAGES_INVALID_CALL = [
# Expected outputs
FUNC_CALC = "calculator"
FUNC_ARGS_CALC = '{"expression":"123 + 456"}'
FUNC_TIME = "get_time"
FUNC_ARGS_TIME = '{"city": "New York"}'
# ==========================================================
@@ -0,0 +1,77 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Claude Code's attribution header in chat completions system messages."""
from unittest.mock import MagicMock
import pytest
from vllm.entrypoints.chat_utils import _parse_chat_message_content
HEADER = (
"x-anthropic-billing-header: cc_version=2.1.220.300; "
"cc_entrypoint=claude-desktop; cch=7adab;"
)
def _parse(message, content_format="openai"):
tracker = MagicMock()
tracker.create_parser.return_value.mm_placeholder_storage.return_value = {}
return _parse_chat_message_content(
message, tracker, content_format, interleave_strings=False
)
@pytest.mark.skip_global_cleanup
@pytest.mark.parametrize(
"header",
[
HEADER,
"x-anthropic-billing-header: cc_version=2.1.286.99b; cc_entrypoint=sdk-cli;",
],
)
def test_system_header_part_is_dropped(header):
(msg,) = _parse(
{
"role": "system",
"content": [
{"type": "text", "text": header},
{"type": "text", "text": "You are a Claude agent."},
],
}
)
assert msg["content"] == [{"type": "text", "text": "You are a Claude agent."}]
@pytest.mark.skip_global_cleanup
def test_system_header_line_is_stripped_from_string():
(msg,) = _parse(
{"role": "system", "content": HEADER + "\nYou are a Claude agent."},
content_format="string",
)
assert msg["content"] == "You are a Claude agent."
@pytest.mark.skip_global_cleanup
def test_requests_differing_only_in_cch_parse_identically():
def system(cch):
return {
"role": "system",
"content": HEADER.replace("7adab", cch) + "\nYou are a Claude agent.",
}
assert _parse(system("7adab")) == _parse(system("986dd"))
@pytest.mark.skip_global_cleanup
@pytest.mark.parametrize(
"message",
[
{"role": "system", "content": "You are a Claude agent."},
{"role": "system", "content": "Quote: " + HEADER},
{"role": "user", "content": HEADER + "Hello"},
],
)
def test_other_content_is_unchanged(message):
(msg,) = _parse(message, content_format="string")
assert msg["content"] == message["content"]
+18 -16
View File
@@ -17,6 +17,7 @@ from vllm.config import ModelConfig
from vllm.entrypoints.chat_utils import (
MEDIA_CONNECTOR_REGISTRY,
AsyncMultiModalItemTracker,
ChatCompletionMessageParam,
ConversationMessage,
_load_embeds_dict,
_parse_metadata_array,
@@ -769,7 +770,9 @@ async def test_text_only_chat_does_not_initialize_media_connector(
):
load_connector = MagicMock()
monkeypatch.setattr(MEDIA_CONNECTOR_REGISTRY, "load", load_connector)
messages = [{"role": "user", "content": "Who are you?"}]
messages: list[ChatCompletionMessageParam] = [
{"role": "user", "content": "Who are you?"}
]
parse_chat_messages(
messages,
@@ -2457,7 +2460,7 @@ def test_parse_chat_messages_image_vision_chunk(
kimi_k2_5_model_config,
image_url,
):
messages = [
messages: list[ChatCompletionMessageParam] = [
{
"role": "user",
"content": [
@@ -2493,7 +2496,7 @@ def test_parse_chat_messages_video_vision_chunk(
kimi_k2_5_model_config,
video_url,
):
messages = [
messages: list[ChatCompletionMessageParam] = [
{
"role": "user",
"content": [
@@ -2530,7 +2533,7 @@ def test_parse_chat_messages_image_vision_chunk_with_uuid(
image_url,
):
image_uuid = "image_123"
messages = [
messages: list[ChatCompletionMessageParam] = [
{
"role": "user",
"content": [
@@ -2568,7 +2571,7 @@ def test_parse_chat_messages_video_vision_chunk_with_uuid(
video_url,
):
video_uuid = "video_456"
messages = [
messages: list[ChatCompletionMessageParam] = [
{
"role": "user",
"content": [
@@ -2606,7 +2609,7 @@ def test_parse_chat_messages_mixed_vision_chunk(
image_url,
video_url,
):
messages = [
messages: list[ChatCompletionMessageParam] = [
{
"role": "user",
"content": [
@@ -2655,7 +2658,7 @@ def test_parse_chat_messages_mixed_vision_chunk_with_uuid(
):
image_uuid = "image_123"
video_uuid = "video_456"
messages = [
messages: list[ChatCompletionMessageParam] = [
{
"role": "user",
"content": [
@@ -2707,7 +2710,7 @@ async def test_parse_chat_messages_mixed_vision_chunk_async(
image_url,
video_url,
):
messages = [
messages: list[ChatCompletionMessageParam] = [
{
"role": "user",
"content": [
@@ -2757,7 +2760,7 @@ async def test_parse_chat_messages_mixed_vision_chunk_with_uuid_async(
):
image_uuid = "image_123"
video_uuid = "video_456"
messages = [
messages: list[ChatCompletionMessageParam] = [
{
"role": "user",
"content": [
@@ -2808,7 +2811,7 @@ async def test_parse_chat_messages_image_vision_chunk_async(
kimi_k2_5_model_config,
image_url,
):
messages = [
messages: list[ChatCompletionMessageParam] = [
{
"role": "user",
"content": [
@@ -2845,7 +2848,7 @@ async def test_parse_chat_messages_video_vision_chunk_async(
kimi_k2_5_model_config,
video_url,
):
messages = [
messages: list[ChatCompletionMessageParam] = [
{
"role": "user",
"content": [
@@ -2883,7 +2886,7 @@ async def test_parse_chat_messages_image_vision_chunk_with_uuid_async(
image_url,
):
image_uuid = "image_123"
messages = [
messages: list[ChatCompletionMessageParam] = [
{
"role": "user",
"content": [
@@ -2922,7 +2925,7 @@ async def test_parse_chat_messages_video_vision_chunk_with_uuid_async(
video_url,
):
video_uuid = "video_456"
messages = [
messages: list[ChatCompletionMessageParam] = [
{
"role": "user",
"content": [
@@ -2997,8 +3000,7 @@ async def test_resolve_items_runs_modalities_concurrently_and_preserves_order():
finally:
active_fetches -= 1
tracker = AsyncMultiModalItemTracker(MagicMock())
tracker._model_config.is_multimodal_model = True
tracker = AsyncMultiModalItemTracker(MagicMock(is_multimodal_model=True))
tracker.__dict__["mm_processor"] = MagicMock()
tracker._items_by_modality["video"] = [
lambda: _fetch("video-0", 0.02),
@@ -3192,5 +3194,5 @@ def test_validate_chat_template_rejects_invalid_type():
with pytest.raises(
VLLMValidationError, match="not a valid chat template type"
) as exc_info:
validate_chat_template(123) # type: ignore[arg-type]
validate_chat_template(123)
assert exc_info.value.parameter == "chat_template"
@@ -0,0 +1,96 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Malformed bodies on RL dev routes are 400s that never reach the engine."""
import json
from types import SimpleNamespace
import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
from vllm.entrypoints.serve.dev.rlhf.api_router import router as rlhf_router
from vllm.entrypoints.serve.dev.rpc.api_router import router as rpc_router
from vllm.entrypoints.serve.exception_handling.register import init_exception_handler
pytestmark = pytest.mark.cpu_test
class Engine:
def __init__(self):
self.calls: list[str] = []
def __getattr__(self, name):
async def call(*args, **kwargs):
self.calls.append(name)
return call
@pytest.fixture
def client():
app = FastAPI()
app.include_router(rlhf_router)
app.include_router(rpc_router)
init_exception_handler(app)
app.state.engine_client = Engine()
app.state.args = SimpleNamespace(log_error_stack=False)
return TestClient(app)
def post(client, path, body):
return client.post(
path, content=json.dumps(body), headers={"content-type": "application/json"}
)
@pytest.mark.parametrize(
"path",
[
"/abort_requests",
"/init_weight_transfer_engine",
"/update_weights",
"/collective_rpc",
],
)
@pytest.mark.parametrize("body", [[], 1, "x", None])
def test_non_object_body_is_rejected(client, path, body):
assert post(client, path, body).status_code == 400
assert client.app.state.engine_client.calls == []
@pytest.mark.parametrize(
"path,body",
[
("/init_weight_transfer_engine", {}),
("/init_weight_transfer_engine", {"init_info": []}),
("/init_weight_transfer_engine", {"init_info": 1}),
("/update_weights", {}),
("/update_weights", {"update_info": 1}),
("/update_weights", {"update_info": [1, 2]}),
("/update_weights", {"update_info": "x"}),
],
)
def test_invalid_field_is_rejected_before_the_engine(client, path, body):
# Reaching the engine would also abort an in-progress weight update.
assert post(client, path, body).status_code == 400
assert client.app.state.engine_client.calls == []
@pytest.mark.parametrize(
"path,body,engine_call",
[
("/abort_requests", {"request_ids": ["a"]}, "abort"),
(
"/init_weight_transfer_engine",
{"init_info": {}},
"init_weight_transfer_engine",
),
("/update_weights", {"update_info": {"names": []}}, "update_weights"),
("/update_weights", {"update_info": [{}, {}]}, "update_weights"),
("/collective_rpc", {"method": "m"}, "collective_rpc"),
],
)
def test_valid_body_reaches_the_engine(client, path, body, engine_call):
assert post(client, path, body).status_code == 200
assert client.app.state.engine_client.calls == [engine_call]
@@ -0,0 +1,188 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
import os
import subprocess
import sys
from contextlib import nullcontext
from pathlib import Path
from types import SimpleNamespace
import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
from prometheus_client import CollectorRegistry
from vllm.entrypoints.serve.dev.rlhf import api_router
from vllm.entrypoints.serve.dev.rlhf import metrics as rlhf_metrics
from vllm.entrypoints.serve.dev.rlhf.metrics import WeightOperationMetrics
from vllm.entrypoints.serve.exception_handling.register import init_exception_handler
from vllm.v1.metrics.prometheus import get_prometheus_registry
pytestmark = [pytest.mark.cpu_test, pytest.mark.skip_global_cleanup]
DURATION = "vllm:rl_weight_update_operation_duration_seconds_count"
IN_FLIGHT = "vllm:rl_weight_update_operations_in_flight"
def sample(registry, name, operation):
return registry.get_sample_value(name, {"operation": operation})
class Request:
def __init__(self, engine):
self.app = SimpleNamespace(state=SimpleNamespace(engine_client=engine))
class Engine:
"""Holds every engine call open until the test releases it."""
def __init__(self):
self.started = asyncio.Event()
self.release = asyncio.Event()
def __getattr__(self, name):
async def call(*args, **kwargs):
self.started.set()
await self.release.wait()
return call
@pytest.fixture
def registry(monkeypatch):
registry = CollectorRegistry()
monkeypatch.setattr(rlhf_metrics, "_metrics", WeightOperationMetrics(registry))
return registry
@pytest.mark.parametrize("fail", [False, True])
def test_record_releases_in_flight_and_observes_duration(registry, fail):
metrics = rlhf_metrics.weight_operation_metrics()
with (
pytest.raises(RuntimeError) if fail else nullcontext(),
metrics.record("update"),
):
assert sample(registry, IN_FLIGHT, "update") == 1
if fail:
raise RuntimeError
assert sample(registry, IN_FLIGHT, "update") == 0
assert sample(registry, DURATION, "update") == 1
@pytest.mark.asyncio
@pytest.mark.parametrize(
"operation,route,body",
[
("init", api_router.init_weight_transfer_engine, {"init_info": {}}),
("start", api_router.start_weight_update, None),
("start_draft", api_router.start_draft_weight_update, None),
("update", api_router.update_weights, {"update_info": {}}),
("finish", api_router.finish_weight_update, None),
],
)
async def test_route_records_its_operation(registry, operation, route, body):
engine = Engine()
task = asyncio.create_task(route(Request(engine), **(body or {})))
await engine.started.wait()
assert sample(registry, IN_FLIGHT, operation) == 1
engine.release.set()
await task
assert sample(registry, IN_FLIGHT, operation) == 0
assert sample(registry, DURATION, operation) == 1
@pytest.mark.asyncio
async def test_cancelled_route_releases_in_flight(registry):
engine = Engine()
task = asyncio.create_task(
api_router.update_weights(Request(engine), update_info={})
)
await engine.started.wait()
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
assert sample(registry, IN_FLIGHT, "update") == 0
assert sample(registry, DURATION, "update") == 1
HOLD_OPERATION = """
import sys
from vllm.entrypoints.serve.dev.rlhf.metrics import weight_operation_metrics
with weight_operation_metrics().record("update"):
print("entered", flush=True)
sys.stdin.readline()
"""
def test_in_flight_is_summed_across_api_server_processes(tmp_path, monkeypatch):
"""Two API-server processes each hold an operation; /metrics sums them."""
monkeypatch.setenv("PROMETHEUS_MULTIPROC_DIR", str(tmp_path))
servers = [
subprocess.Popen(
[sys.executable, "-c", HOLD_OPERATION],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
text=True,
env=os.environ.copy(),
)
for _ in range(2)
]
try:
for server in servers:
assert server.stdout is not None
assert server.stdout.readline().strip() == "entered"
assert sample(get_prometheus_registry(), IN_FLIGHT, "update") == 2
finally:
for server in servers:
server.communicate("\n", timeout=60)
scraped = get_prometheus_registry()
assert sample(scraped, IN_FLIGHT, "update") == 0
assert sample(scraped, DURATION, "update") == 2
def test_rejected_request_is_not_recorded(registry):
app = FastAPI()
app.include_router(api_router.router)
init_exception_handler(app)
app.state.engine_client = Engine()
app.state.args = SimpleNamespace(log_error_stack=False)
response = TestClient(app).post("/update_weights", json={})
assert response.status_code == 400
assert sample(registry, DURATION, "update") is None
def test_default_recorder_is_created_lazily_on_the_default_registry():
"""The default recorder registers on the process registry only when first used.
Runs in a fresh interpreter: the registry is process-global, so a route test
elsewhere in the session may already have registered these collectors, and
"not registered yet" would then be unobservable.
"""
probe = """
from prometheus_client import REGISTRY
from vllm.entrypoints.serve.dev.rlhf import metrics as rlhf_metrics
NAMES = [
"vllm:rl_weight_update_operation_duration_seconds",
"vllm:rl_weight_update_operations_in_flight",
]
assert rlhf_metrics._metrics is None
assert not set(NAMES) & set(REGISTRY._names_to_collectors)
metrics = rlhf_metrics.weight_operation_metrics()
assert REGISTRY._names_to_collectors[NAMES[0]] is metrics.duration
assert REGISTRY._names_to_collectors[NAMES[1]] is metrics.in_flight
assert rlhf_metrics.weight_operation_metrics() is metrics
print("lazy registration ok")
"""
result = subprocess.run(
[sys.executable, "-c", probe],
cwd=Path(__file__).parents[3],
capture_output=True,
text=True,
)
assert result.returncode == 0, result.stderr
assert "lazy registration ok" in result.stdout
@@ -0,0 +1,17 @@
model_name: "deepseek-ai/DeepSeek-R1"
accuracy_threshold: 0.95
num_questions: 1319
num_fewshot: 5
startup_max_wait_seconds: 1800
env:
VLLM_ROCM_USE_AITER: "1"
VLLM_ROCM_USE_AITER_MLA: "1"
VLLM_ROCM_USE_AITER_MOE: "1"
server_args: >-
--enforce-eager
--max-model-len 4096
--data-parallel-size 4
--enable-expert-parallel
--attention-backend ROCM_AITER_MLA
--moe-backend aiter
--speculative-config '{"method":"mtp","num_speculative_tokens":3}'
@@ -0,0 +1,17 @@
model_name: "deepseek-ai/DeepSeek-R1"
accuracy_threshold: 0.95
num_questions: 1319
num_fewshot: 5
startup_max_wait_seconds: 1800
env:
VLLM_ROCM_USE_AITER: "1"
VLLM_ROCM_USE_AITER_MLA: "1"
VLLM_ROCM_USE_AITER_MOE: "1"
server_args: >-
--enforce-eager
--max-model-len 4096
--tensor-parallel-size 4
--enable-expert-parallel
--attention-backend ROCM_AITER_MLA
--moe-backend aiter
--speculative-config '{"method":"mtp","num_speculative_tokens":3}'
@@ -0,0 +1,17 @@
model_name: "deepseek-ai/DeepSeek-V3.2"
accuracy_threshold: 0.95
num_questions: 1319
num_fewshot: 5
startup_max_wait_seconds: 1200
env:
VLLM_ROCM_USE_AITER: "1"
VLLM_ROCM_USE_AITER_MLA: "1"
VLLM_ROCM_USE_AITER_MOE: "1"
server_args: >-
--enforce-eager
--max-model-len 4096
--data-parallel-size 4
--enable-expert-parallel
--attention-backend ROCM_AITER_MLA_SPARSE
--moe-backend aiter
--speculative-config '{"method":"mtp","num_speculative_tokens":3}'
@@ -0,0 +1,17 @@
model_name: "deepseek-ai/DeepSeek-V3.2"
accuracy_threshold: 0.95
num_questions: 1319
num_fewshot: 5
startup_max_wait_seconds: 1200
env:
VLLM_ROCM_USE_AITER: "1"
VLLM_ROCM_USE_AITER_MLA: "1"
VLLM_ROCM_USE_AITER_MOE: "1"
server_args: >-
--enforce-eager
--max-model-len 4096
--tensor-parallel-size 4
--enable-expert-parallel
--attention-backend ROCM_AITER_MLA_SPARSE
--moe-backend aiter
--speculative-config '{"method":"mtp","num_speculative_tokens":3}'
@@ -0,0 +1,15 @@
model_name: "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16"
accuracy_threshold: 0.93
num_questions: 1319
num_fewshot: 5
startup_max_wait_seconds: 1200
env:
VLLM_ROCM_USE_AITER: "1"
server_args: >-
--enforce-eager
--max-model-len 4096
--tensor-parallel-size 4
--enable-expert-parallel
--mamba-backend triton
--moe-backend triton
--speculative-config '{"method":"mtp","num_speculative_tokens":5}'
@@ -0,0 +1,15 @@
model_name: "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8"
accuracy_threshold: 0.93
num_questions: 1319
num_fewshot: 5
startup_max_wait_seconds: 1200
env:
VLLM_ROCM_USE_AITER: "1"
server_args: >-
--enforce-eager
--max-model-len 4096
--tensor-parallel-size 4
--enable-expert-parallel
--mamba-backend triton
--moe-backend triton
--speculative-config '{"method":"mtp","num_speculative_tokens":5}'
@@ -0,0 +1,18 @@
model_name: "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4"
accuracy_threshold: 0.93
num_questions: 1319
num_fewshot: 5
max_concurrency: 16
rocm_request_timeout_seconds: 1800
min_acceptance_length: 2.0
startup_max_wait_seconds: 1200
server_args: >-
--enforce-eager
--max-model-len 4096
--max-num-seqs 16
--max-num-batched-tokens 32768
--gpu-memory-utilization 0.5
--tensor-parallel-size 2
--enable-expert-parallel
--mamba-backend triton
--speculative-config '{"method":"mtp","num_speculative_tokens":5}'
@@ -0,0 +1,17 @@
model_name: "nm-testing/Qwen3-Next-80B-A3B-Instruct-NVFP4"
accuracy_threshold: 0.75
num_questions: 1319
num_fewshot: 5
max_concurrency: 16
rocm_request_timeout_seconds: 1800
server_args: >-
--enforce-eager
--max-model-len 4096
--max-num-seqs 16
--max-num-batched-tokens 32768
--gpu-memory-utilization 0.5
--tensor-parallel-size 2
--enable-expert-parallel
--all2all-backend allgather_reducescatter
--attention-backend TRITON_ATTN
--no-async-scheduling
@@ -0,0 +1,19 @@
model_name: "nvidia/Qwen3-Next-80B-A3B-Instruct-NVFP4"
accuracy_threshold: 0.85
num_questions: 1319
num_fewshot: 5
max_concurrency: 16
rocm_request_timeout_seconds: 1800
min_acceptance_length: 1.5
server_args: >-
--enforce-eager
--max-model-len 4096
--max-num-seqs 16
--max-num-batched-tokens 32768
--gpu-memory-utilization 0.5
--tensor-parallel-size 2
--enable-expert-parallel
--all2all-backend allgather_reducescatter
--attention-backend TRITON_ATTN
--no-async-scheduling
--speculative-config '{"method":"qwen3_next_mtp","num_speculative_tokens":1}'
@@ -0,0 +1,5 @@
# Exact B200 compressed-tensors checkpoint; it does not contain MTP weights.
Qwen3-Next-80B-A3B-NVFP4-EP2-CT-MI355.yaml
# MTP-capable ModelOpt checkpoint exercises NVFP4 EP2 with MTP1.
Qwen3-Next-80B-A3B-NVFP4-EP2-MTP-MI355.yaml
Nemotron-3-Super-120B-A12B-NVFP4-MI355.yaml
@@ -1,2 +1,5 @@
Qwen3-0.6B-FP8.yaml
Qwen3-1.7B-MXFP4.yaml
Qwen3-30B-A3B-NVFP4.yaml
Laguna-XS.2-NVFP4.yaml
Laguna-XS-2.1-INT4.yaml
@@ -0,0 +1 @@
DeepSeek-R1-DP_MI355.yaml
@@ -0,0 +1 @@
DeepSeek-R1-TP_MI355.yaml
@@ -0,0 +1 @@
DeepSeek-V3.2-DP_MI355.yaml
@@ -0,0 +1 @@
DeepSeek-V3.2-TP_MI355.yaml
@@ -0,0 +1,2 @@
Nemotron-3-Super-120B-A12B-BF16_MI355.yaml
Nemotron-3-Super-120B-A12B-FP8_MI355.yaml

Some files were not shown because too many files have changed in this diff Show More