Merge remote-tracking branch 'upstream/main' into patches
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,6 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import gc
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
@@ -233,6 +235,98 @@ def test_discard_tags():
|
||||
assert torch.allclose(weights, torch.ones_like(weights))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tag", ["workspace", None], ids=["workspace", "default"])
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
def test_selective_wake_restores_internal_tags(tag):
|
||||
"""A selective wake defers only weights/kv_cache; other tags always wake."""
|
||||
allocator = get_mem_allocator_instance()
|
||||
with allocator.use_memory_pool(tag):
|
||||
internal = torch.empty(16 << 20, dtype=torch.uint8, device=DEVICE_TYPE)
|
||||
with allocator.use_memory_pool("kv_cache"):
|
||||
kv = torch.empty(32 << 20, dtype=torch.uint8, device=DEVICE_TYPE)
|
||||
|
||||
allocator.sleep(offload_tags=())
|
||||
assert mapped_usage(allocator) == 0
|
||||
|
||||
allocator.wake_up(tags=["weights"])
|
||||
assert mapped_usage(allocator) == internal.nbytes
|
||||
allocator.wake_up(tags=["kv_cache"])
|
||||
assert mapped_usage(allocator) == internal.nbytes + kv.nbytes
|
||||
|
||||
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
@pytest.mark.skipif(current_platform.is_xpu(), reason="Uses the CuMem allocator")
|
||||
def test_workspace_scratch_discarded_on_sleep():
|
||||
"""Workspace scratch lives in the sleep-mode pool: sleep discards it, any
|
||||
(selective) wake remaps it at the same address so captured graphs replay,
|
||||
and a buffer outgrown by a resize stays usable until it is released."""
|
||||
from vllm.device_allocator.sleep_mode_backend import CuMemBackend
|
||||
from vllm.v1.worker.workspace import WorkspaceManager
|
||||
|
||||
allocator = get_mem_allocator_instance()
|
||||
manager = WorkspaceManager(
|
||||
torch.device(DEVICE_TYPE),
|
||||
alloc_context=lambda: allocator.use_memory_pool("workspace"),
|
||||
)
|
||||
(small,) = manager.get_simultaneous(((32 << 20,), torch.uint8))
|
||||
# Resize while a view of the old buffer is alive: the tag is re-entered.
|
||||
(scratch,) = manager.get_simultaneous(((64 << 20,), torch.uint8))
|
||||
assert len(allocator.allocator_and_pools["workspace"]) == 2
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
scratch.add_(1)
|
||||
|
||||
backend = CuMemBackend()
|
||||
backend.suspend(level=1)
|
||||
assert mapped_usage(allocator) == 0
|
||||
# Discarded, not copied.
|
||||
assert all(
|
||||
d.tag == "workspace" and d.cpu_backup_tensor is None
|
||||
for d in allocator.pointer_to_data.values()
|
||||
)
|
||||
|
||||
backend.resume(tags=["kv_cache"])
|
||||
assert mapped_usage(allocator) == small.nbytes + scratch.nbytes
|
||||
scratch.zero_()
|
||||
graph.replay()
|
||||
assert int(scratch.sum()) == scratch.numel()
|
||||
small.fill_(1)
|
||||
|
||||
# The next resize releases the outgrown buffer; wake does not remap it.
|
||||
del small
|
||||
(big,) = manager.get_simultaneous(((128 << 20,), torch.uint8))
|
||||
assert mapped_usage(allocator) == scratch.nbytes + big.nbytes
|
||||
backend.suspend(level=1)
|
||||
backend.resume(tags=["weights"])
|
||||
assert mapped_usage(allocator) == scratch.nbytes + big.nbytes
|
||||
|
||||
|
||||
@pytest.mark.parametrize("free_x_early", [True, False], ids=["x-freed", "x-alive"])
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
@pytest.mark.skipif(current_platform.is_xpu(), reason="Uses the CuMem allocator")
|
||||
def test_reentered_tag(free_x_early):
|
||||
"""Enter a tag twice (two workspace ubatches): freeing X returns its memory
|
||||
by the next entry, without touching Y or breaking sleep/wake."""
|
||||
allocator = get_mem_allocator_instance()
|
||||
nbytes = 64 << 20
|
||||
with allocator.use_memory_pool("t"):
|
||||
xs = [torch.empty(nbytes, dtype=torch.uint8, device=DEVICE_TYPE)]
|
||||
if free_x_early:
|
||||
xs.clear()
|
||||
with allocator.use_memory_pool("t"):
|
||||
y = torch.empty(nbytes, dtype=torch.uint8, device=DEVICE_TYPE)
|
||||
xs.clear()
|
||||
gc.collect()
|
||||
with allocator.use_memory_pool("t"):
|
||||
pass
|
||||
assert mapped_usage(allocator) == nbytes
|
||||
y.fill_(1)
|
||||
allocator.sleep(offload_tags="t")
|
||||
allocator.wake_up()
|
||||
assert int(y.sum()) == y.numel()
|
||||
allocator.release_pools()
|
||||
|
||||
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
@pytest.mark.skipif(current_platform.is_xpu(), reason="Uses the CuMem allocator")
|
||||
def test_level2_discards_ordinary_tensor_with_weights_tag():
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import asyncio
|
||||
import math
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -9,6 +10,7 @@ import torch
|
||||
import vllm.device_allocator.cumem as cumem
|
||||
import vllm.envs as envs
|
||||
from vllm import LLM, AsyncEngineArgs, AsyncLLMEngine, SamplingParams
|
||||
from vllm.model_executor.model_loader import get_model_loader
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.mem_constants import GiB_bytes
|
||||
|
||||
@@ -232,6 +234,58 @@ def test_deep_sleep_async(gpu_memory_cleared):
|
||||
asyncio.run(test())
|
||||
|
||||
|
||||
def _load_weights(worker) -> None:
|
||||
# Not reload_weights(): its layerwise finalize re-derives the KV scales.
|
||||
model = worker.model_runner.get_model()
|
||||
loader = get_model_loader(worker.load_config)
|
||||
model.load_weights(loader.get_all_weights(worker.model_config, model))
|
||||
|
||||
|
||||
def _attn_kv_scales(model) -> dict[str, list[float]]:
|
||||
return {
|
||||
f"{name}.{attr}": getattr(module, attr).flatten().tolist()
|
||||
for name, module in model.named_modules()
|
||||
for attr in ("_q_scale", "_k_scale", "_v_scale")
|
||||
if isinstance(getattr(module, attr, None), torch.Tensor)
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Segfaults on ROCm.")
|
||||
@requires_fp8
|
||||
@create_new_process_for_each_test()
|
||||
def test_deep_sleep_compressed_tensors_kv_scales(
|
||||
monkeypatch: pytest.MonkeyPatch, gpu_memory_cleared
|
||||
):
|
||||
"""KV scales must survive level-2 sleep plus a weights reload (else NaN)."""
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
llm = LLM(
|
||||
"nm-testing/TinyLlama-1.1B-Chat-v1.0-kvcache-fp8-tensor",
|
||||
enable_sleep_mode=True,
|
||||
kv_cache_dtype="fp8",
|
||||
enforce_eager=True,
|
||||
gpu_memory_utilization=0.6,
|
||||
)
|
||||
sampling_params = SamplingParams(temperature=0, max_tokens=10, logprobs=0)
|
||||
scales = llm.apply_model(_attn_kv_scales)[0]
|
||||
assert any(v != 1.0 for values in scales.values() for v in values)
|
||||
expected = llm.generate("How are you?", sampling_params)[0].outputs[0]
|
||||
|
||||
llm.sleep(level=2)
|
||||
llm.wake_up(tags=["weights"])
|
||||
llm.collective_rpc(_load_weights)
|
||||
llm.wake_up(tags=["kv_cache"])
|
||||
|
||||
assert llm.apply_model(_attn_kv_scales)[0] == scales
|
||||
actual = llm.generate("How are you?", sampling_params)[0].outputs[0]
|
||||
assert actual.token_ids == expected.token_ids
|
||||
logprobs = [
|
||||
[lp[t].logprob for lp, t in zip(o.logprobs, o.token_ids)]
|
||||
for o in (expected, actual)
|
||||
]
|
||||
assert all(math.isfinite(x) for x in logprobs[1])
|
||||
assert logprobs[0] == logprobs[1]
|
||||
|
||||
|
||||
@requires_fp8
|
||||
def test_deep_sleep_fp8_kvcache_mrv1(
|
||||
monkeypatch: pytest.MonkeyPatch, gpu_memory_cleared
|
||||
|
||||
@@ -6,6 +6,7 @@ from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.benchmarks.sweep import serve as sweep_serve
|
||||
from vllm.benchmarks.sweep.param_sweep import ParameterSweep, ParameterSweepItem
|
||||
|
||||
|
||||
@@ -115,7 +116,7 @@ class TestParameterSweep:
|
||||
|
||||
def test_from_records_list(self):
|
||||
"""Test creating ParameterSweep from a list of records."""
|
||||
records = [
|
||||
records: list[dict[str, object]] = [
|
||||
{"max_tokens": 100, "temperature": 0.7},
|
||||
{"max_tokens": 200, "temperature": 0.9},
|
||||
]
|
||||
@@ -126,7 +127,7 @@ class TestParameterSweep:
|
||||
|
||||
def test_read_from_dict(self):
|
||||
"""Test creating ParameterSweep from a dict format."""
|
||||
data = {
|
||||
data: dict[str, dict[str, object]] = {
|
||||
"experiment1": {"max_tokens": 100, "temperature": 0.7},
|
||||
"experiment2": {"max_tokens": 200, "temperature": 0.9},
|
||||
}
|
||||
@@ -148,7 +149,7 @@ class TestParameterSweep:
|
||||
|
||||
def test_read_json_list_format(self):
|
||||
"""Test reading JSON file with list format."""
|
||||
records = [
|
||||
records: list[dict[str, object]] = [
|
||||
{"max_tokens": 100, "temperature": 0.7},
|
||||
{"max_tokens": 200, "temperature": 0.9},
|
||||
]
|
||||
@@ -167,7 +168,7 @@ class TestParameterSweep:
|
||||
|
||||
def test_read_json_dict_format(self):
|
||||
"""Test reading JSON file with dict format."""
|
||||
data = {
|
||||
data: dict[str, dict[str, object]] = {
|
||||
"experiment1": {"max_tokens": 100, "temperature": 0.7},
|
||||
"experiment2": {"max_tokens": 200, "temperature": 0.9},
|
||||
}
|
||||
@@ -189,7 +190,7 @@ class TestParameterSweep:
|
||||
def test_unique_benchmark_names_validation(self):
|
||||
"""Test that duplicate _benchmark_name values raise an error."""
|
||||
# Test with duplicate names in list format
|
||||
records = [
|
||||
records: list[dict[str, object]] = [
|
||||
{"_benchmark_name": "exp1", "max_tokens": 100},
|
||||
{"_benchmark_name": "exp1", "max_tokens": 200},
|
||||
]
|
||||
@@ -199,7 +200,7 @@ class TestParameterSweep:
|
||||
|
||||
def test_unique_benchmark_names_multiple_duplicates(self):
|
||||
"""Test validation with multiple duplicate names."""
|
||||
records = [
|
||||
records: list[dict[str, object]] = [
|
||||
{"_benchmark_name": "exp1", "max_tokens": 100},
|
||||
{"_benchmark_name": "exp1", "max_tokens": 200},
|
||||
{"_benchmark_name": "exp2", "max_tokens": 300},
|
||||
@@ -211,7 +212,7 @@ class TestParameterSweep:
|
||||
|
||||
def test_no_benchmark_names_allowed(self):
|
||||
"""Test that records without _benchmark_name are allowed."""
|
||||
records = [
|
||||
records: list[dict[str, object]] = [
|
||||
{"max_tokens": 100, "temperature": 0.7},
|
||||
{"max_tokens": 200, "temperature": 0.9},
|
||||
]
|
||||
@@ -220,7 +221,7 @@ class TestParameterSweep:
|
||||
|
||||
def test_mixed_benchmark_names_allowed(self):
|
||||
"""Test that mixing records with and without _benchmark_name is allowed."""
|
||||
records = [
|
||||
records: list[dict[str, object]] = [
|
||||
{"_benchmark_name": "exp1", "max_tokens": 100},
|
||||
{"max_tokens": 200, "temperature": 0.9},
|
||||
]
|
||||
@@ -247,3 +248,128 @@ class TestParameterSweepItemKeyNormalization:
|
||||
# The prefix (compilation_config) gets converted to hyphens,
|
||||
# but the suffix (some_nested_param) is preserved
|
||||
assert any("compilation-config.some_nested_param" in arg for arg in cmd)
|
||||
|
||||
|
||||
def test_run_comb_excludes_warmup_from_measured_results(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
):
|
||||
calls: list[tuple[int, int, str]] = []
|
||||
|
||||
def fake_run_benchmark(
|
||||
server,
|
||||
bench_cmd,
|
||||
*,
|
||||
serve_overrides,
|
||||
bench_overrides,
|
||||
run_number,
|
||||
output_path,
|
||||
dry_run,
|
||||
):
|
||||
calls.append(
|
||||
(run_number, int(bench_overrides["num_prompts"]), output_path.name)
|
||||
)
|
||||
return {"run_number": run_number}
|
||||
|
||||
monkeypatch.setattr(sweep_serve, "run_benchmark", fake_run_benchmark)
|
||||
base_path = tmp_path / "combination"
|
||||
base_path.mkdir()
|
||||
|
||||
measured = sweep_serve.run_comb(
|
||||
None,
|
||||
[],
|
||||
serve_comb=ParameterSweepItem(),
|
||||
bench_comb=ParameterSweepItem({"num_prompts": 320}),
|
||||
link_vars=[],
|
||||
base_path=base_path,
|
||||
num_runs=2,
|
||||
warmup_num_prompts=32,
|
||||
dry_run=False,
|
||||
)
|
||||
|
||||
assert calls == [
|
||||
(-1, 32, "warmup.json"),
|
||||
(0, 320, "run=0.json"),
|
||||
(1, 320, "run=1.json"),
|
||||
]
|
||||
assert measured == [{"run_number": 0}, {"run_number": 1}]
|
||||
|
||||
|
||||
def test_run_comb_continue_on_error_keeps_later_runs(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
):
|
||||
calls: list[int] = []
|
||||
|
||||
def fake_run_benchmark(
|
||||
server,
|
||||
bench_cmd,
|
||||
*,
|
||||
serve_overrides,
|
||||
bench_overrides,
|
||||
run_number,
|
||||
output_path,
|
||||
dry_run,
|
||||
):
|
||||
calls.append(run_number)
|
||||
if run_number == 0:
|
||||
raise RuntimeError("synthetic run failure")
|
||||
return {"run_number": run_number}
|
||||
|
||||
monkeypatch.setattr(sweep_serve, "run_benchmark", fake_run_benchmark)
|
||||
base_path = tmp_path / "combination"
|
||||
base_path.mkdir()
|
||||
|
||||
measured = sweep_serve.run_comb(
|
||||
None,
|
||||
[],
|
||||
serve_comb=ParameterSweepItem(),
|
||||
bench_comb=ParameterSweepItem({"num_prompts": 100}),
|
||||
link_vars=[],
|
||||
base_path=base_path,
|
||||
num_runs=2,
|
||||
warmup_num_prompts=0,
|
||||
dry_run=False,
|
||||
continue_on_error=True,
|
||||
)
|
||||
|
||||
assert calls == [0, 1]
|
||||
assert measured == [{"run_number": 1}]
|
||||
assert (base_path / "run=0.failure.json").exists()
|
||||
assert (base_path / "summary.json").exists()
|
||||
|
||||
|
||||
def test_run_comb_warmup_default_is_backward_compatible(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
):
|
||||
"""Older callers can omit warmup_num_prompts."""
|
||||
calls: list[int] = []
|
||||
|
||||
def fake_run_benchmark(
|
||||
server,
|
||||
bench_cmd,
|
||||
*,
|
||||
serve_overrides,
|
||||
bench_overrides,
|
||||
run_number,
|
||||
output_path,
|
||||
dry_run,
|
||||
):
|
||||
calls.append(run_number)
|
||||
return {"run_number": run_number}
|
||||
|
||||
monkeypatch.setattr(sweep_serve, "run_benchmark", fake_run_benchmark)
|
||||
base_path = tmp_path / "combination"
|
||||
base_path.mkdir()
|
||||
|
||||
measured = sweep_serve.run_comb(
|
||||
None,
|
||||
[],
|
||||
serve_comb=ParameterSweepItem(),
|
||||
bench_comb=ParameterSweepItem({"num_prompts": 10}),
|
||||
link_vars=[],
|
||||
base_path=base_path,
|
||||
num_runs=1,
|
||||
dry_run=False,
|
||||
)
|
||||
|
||||
assert calls == [0]
|
||||
assert measured == [{"run_number": 0}]
|
||||
|
||||
@@ -13,6 +13,7 @@ import soundfile as sf
|
||||
import vllm.benchmarks.datasets.datasets as datasets_module
|
||||
import vllm.benchmarks.lib.endpoint_request_func as request_func_module
|
||||
from vllm.benchmarks.lib.endpoint_request_func import RequestFuncInput
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
|
||||
@@ -44,6 +45,10 @@ class _CohereNameOnlyTokenizer(_Tokenizer):
|
||||
super().__init__("cohere/some-local-checkpoint")
|
||||
|
||||
|
||||
def _as_tokenizer(tokenizer: _Tokenizer) -> TokenizerLike:
|
||||
return cast(TokenizerLike, tokenizer)
|
||||
|
||||
|
||||
def _write_wav(path: Path, duration_s: float = 0.1, sample_rate: int = 16_000) -> None:
|
||||
num_samples = int(duration_s * sample_rate)
|
||||
sf.write(path, np.zeros(num_samples, dtype=np.float32), sample_rate)
|
||||
@@ -109,7 +114,7 @@ def test_asr_dataset_sample_handles_local_audio_paths(tmp_path: Path) -> None:
|
||||
]
|
||||
|
||||
samples = dataset.sample(
|
||||
tokenizer=_Tokenizer(),
|
||||
tokenizer=_as_tokenizer(_Tokenizer()),
|
||||
num_requests=1,
|
||||
output_len=32,
|
||||
asr_min_audio_len_sec=0.0,
|
||||
@@ -146,7 +151,7 @@ def test_asr_dataset_sample_handles_embedded_audio_bytes(
|
||||
]
|
||||
|
||||
samples = dataset.sample(
|
||||
tokenizer=_Tokenizer(),
|
||||
tokenizer=_as_tokenizer(_Tokenizer()),
|
||||
num_requests=1,
|
||||
output_len=32,
|
||||
asr_min_audio_len_sec=0.0,
|
||||
@@ -240,7 +245,7 @@ def _make_asr_dataset(tmp_path: Path) -> datasets_module.ASRDataset:
|
||||
def test_asr_dataset_cohere_class_name_gets_decoder_prompt(tmp_path: Path) -> None:
|
||||
dataset = _make_asr_dataset(tmp_path)
|
||||
samples = dataset.sample(
|
||||
tokenizer=CohereAsrTokenizer(),
|
||||
tokenizer=_as_tokenizer(CohereAsrTokenizer()),
|
||||
num_requests=1,
|
||||
output_len=32,
|
||||
asr_min_audio_len_sec=0.0,
|
||||
@@ -255,7 +260,7 @@ def test_asr_dataset_cohere_name_or_path_fallback_gets_decoder_prompt(
|
||||
) -> None:
|
||||
dataset = _make_asr_dataset(tmp_path)
|
||||
samples = dataset.sample(
|
||||
tokenizer=_CohereNameOnlyTokenizer(),
|
||||
tokenizer=_as_tokenizer(_CohereNameOnlyTokenizer()),
|
||||
num_requests=1,
|
||||
output_len=32,
|
||||
asr_min_audio_len_sec=0.0,
|
||||
@@ -268,7 +273,7 @@ def test_asr_dataset_cohere_name_or_path_fallback_gets_decoder_prompt(
|
||||
def test_asr_dataset_unknown_tokenizer_gets_empty_prompt(tmp_path: Path) -> None:
|
||||
dataset = _make_asr_dataset(tmp_path)
|
||||
samples = dataset.sample(
|
||||
tokenizer=_Tokenizer(name_or_path="some-other/asr-model"),
|
||||
tokenizer=_as_tokenizer(_Tokenizer(name_or_path="some-other/asr-model")),
|
||||
num_requests=1,
|
||||
output_len=32,
|
||||
asr_min_audio_len_sec=0.0,
|
||||
|
||||
@@ -4,7 +4,7 @@ import json
|
||||
from argparse import Namespace
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from typing import Any, cast
|
||||
|
||||
import pybase64 as base64
|
||||
import pytest
|
||||
@@ -17,6 +17,7 @@ from vllm.benchmarks.lib.endpoint_request_func import (
|
||||
_get_chat_content,
|
||||
_get_chat_messages,
|
||||
)
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
|
||||
@@ -31,6 +32,10 @@ class _Tokenizer:
|
||||
return _TokenizedPrompt(prompt)
|
||||
|
||||
|
||||
def _tokenizer() -> TokenizerLike:
|
||||
return cast(TokenizerLike, _Tokenizer())
|
||||
|
||||
|
||||
def _write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None:
|
||||
with path.open("w") as f:
|
||||
for row in rows:
|
||||
@@ -92,7 +97,7 @@ def test_get_samples_custom_image_cli_path_supports_multi_image_and_content(
|
||||
],
|
||||
)
|
||||
|
||||
samples = get_samples(_args_for_custom_image(jsonl), _Tokenizer())
|
||||
samples = get_samples(_args_for_custom_image(jsonl), _tokenizer())
|
||||
|
||||
assert len(samples) == 2
|
||||
assert samples[0].request_id == "req-0"
|
||||
@@ -106,6 +111,7 @@ def test_get_samples_custom_image_cli_path_supports_multi_image_and_content(
|
||||
assert samples[1].multi_modal_data is None
|
||||
assert isinstance(samples[1].prompt, list)
|
||||
assert samples[1].prompt[0] == {"type": "text", "text": "Now compare "}
|
||||
assert isinstance(samples[1].prompt[1], dict)
|
||||
assert samples[1].prompt[1]["image_url"]["url"] == f"file://{image_c}"
|
||||
|
||||
|
||||
@@ -126,7 +132,7 @@ def test_custom_image_dataset_uses_all_image_files(tmp_path: Path) -> None:
|
||||
|
||||
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
|
||||
samples = dataset.sample(
|
||||
tokenizer=_Tokenizer(),
|
||||
tokenizer=_tokenizer(),
|
||||
num_requests=1,
|
||||
output_len=32,
|
||||
)
|
||||
@@ -171,7 +177,7 @@ def test_custom_image_dataset_preserves_interleaved_content_order(
|
||||
|
||||
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
|
||||
samples = dataset.sample(
|
||||
tokenizer=_Tokenizer(),
|
||||
tokenizer=_tokenizer(),
|
||||
num_requests=1,
|
||||
output_len=32,
|
||||
)
|
||||
@@ -181,14 +187,18 @@ def test_custom_image_dataset_preserves_interleaved_content_order(
|
||||
assert sample.multi_modal_data is None
|
||||
assert sample.prompt_len == 2
|
||||
assert isinstance(sample.prompt, list)
|
||||
assert [part["type"] for part in sample.prompt] == [
|
||||
parts = []
|
||||
for part in sample.prompt:
|
||||
assert isinstance(part, dict)
|
||||
parts.append(part)
|
||||
assert [part["type"] for part in parts] == [
|
||||
"text",
|
||||
"image_url",
|
||||
"text",
|
||||
"image_url",
|
||||
]
|
||||
assert sample.prompt[1]["image_url"]["url"] == f"file://{image_a}"
|
||||
assert sample.prompt[3]["image_url"] == {
|
||||
assert parts[1]["image_url"]["url"] == f"file://{image_a}"
|
||||
assert parts[3]["image_url"] == {
|
||||
"url": f"file://{image_b}",
|
||||
"detail": "low",
|
||||
}
|
||||
@@ -223,7 +233,7 @@ def test_custom_image_dataset_wraps_interleaved_content_for_multimodal_chat(
|
||||
|
||||
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
|
||||
samples = dataset.sample(
|
||||
tokenizer=_Tokenizer(),
|
||||
tokenizer=_tokenizer(),
|
||||
num_requests=1,
|
||||
output_len=32,
|
||||
enable_multimodal_chat=True,
|
||||
@@ -289,7 +299,7 @@ def test_custom_image_dataset_encodes_image_media_when_requested(
|
||||
|
||||
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
|
||||
samples = dataset.sample(
|
||||
tokenizer=_Tokenizer(),
|
||||
tokenizer=_tokenizer(),
|
||||
num_requests=1,
|
||||
output_len=32,
|
||||
ensure_client_side_data=True,
|
||||
@@ -335,7 +345,7 @@ def test_custom_image_dataset_encodes_interleaved_image_media(
|
||||
|
||||
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
|
||||
samples = dataset.sample(
|
||||
tokenizer=_Tokenizer(),
|
||||
tokenizer=_tokenizer(),
|
||||
num_requests=1,
|
||||
output_len=32,
|
||||
ensure_client_side_data=True,
|
||||
@@ -343,9 +353,13 @@ def test_custom_image_dataset_encodes_interleaved_image_media(
|
||||
|
||||
sample = samples[0]
|
||||
assert isinstance(sample.prompt, list)
|
||||
_assert_png_data_url(sample.prompt[1]["image_url"]["url"])
|
||||
_assert_png_data_url(sample.prompt[2]["image_url"]["url"])
|
||||
assert sample.prompt[2]["image_url"]["detail"] == "low"
|
||||
parts = []
|
||||
for part in sample.prompt:
|
||||
assert isinstance(part, dict)
|
||||
parts.append(part)
|
||||
_assert_png_data_url(parts[1]["image_url"]["url"])
|
||||
_assert_png_data_url(parts[2]["image_url"]["url"])
|
||||
assert parts[2]["image_url"]["detail"] == "low"
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
@@ -363,7 +377,7 @@ def test_custom_image_dataset_rejects_invalid_image_media(
|
||||
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
|
||||
with pytest.raises(ValueError, match="Invalid image URL"):
|
||||
dataset.sample(
|
||||
tokenizer=_Tokenizer(),
|
||||
tokenizer=_tokenizer(),
|
||||
num_requests=1,
|
||||
output_len=32,
|
||||
ensure_client_side_data=True,
|
||||
@@ -380,7 +394,7 @@ def test_custom_image_dataset_rejects_invalid_content_part(
|
||||
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
|
||||
with pytest.raises(ValueError, match="type 'text', 'image', or 'image_url'"):
|
||||
dataset.sample(
|
||||
tokenizer=_Tokenizer(),
|
||||
tokenizer=_tokenizer(),
|
||||
num_requests=1,
|
||||
output_len=32,
|
||||
)
|
||||
|
||||
@@ -37,6 +37,7 @@ def random_dataset_params() -> Params:
|
||||
|
||||
def _fingerprint_sample(req: SampleRequest) -> tuple[str, int, int]:
|
||||
"""Project a SampleRequest into a comparable tuple."""
|
||||
assert isinstance(req.prompt, str)
|
||||
return (req.prompt, req.prompt_len, req.expected_output_len)
|
||||
|
||||
|
||||
@@ -153,6 +154,7 @@ def _mm_fingerprint_sample(
|
||||
- count of multimodal items
|
||||
- per-item type and URL prefix (e.g., 'data:image/jpeg;base64,')
|
||||
"""
|
||||
assert isinstance(req.prompt, str)
|
||||
items = req.multi_modal_data or []
|
||||
item_prefixes: list[str] = []
|
||||
for it in items:
|
||||
|
||||
@@ -2,8 +2,11 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from tempfile import NamedTemporaryFile
|
||||
from typing import Any, cast
|
||||
from unittest.mock import Mock
|
||||
|
||||
import cv2
|
||||
import pybase64 as base64
|
||||
@@ -61,11 +64,17 @@ def test_map_config_to_modality(video_dataset: RandomMultiModalDataset):
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_generate_mm_item_video(video_dataset: RandomMultiModalDataset):
|
||||
"""Test generating multimodal items for video configurations."""
|
||||
def test_generate_mm_item_video(
|
||||
video_dataset: RandomMultiModalDataset,
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""Generate a decodable video without leaving temporary files behind."""
|
||||
monkeypatch.setattr(tempfile, "tempdir", str(tmp_path))
|
||||
# Test video item generation
|
||||
video_config = (64, 48, 8) # height, width, num_frames
|
||||
result = video_dataset.generate_mm_item(video_config)
|
||||
assert not list(tmp_path.iterdir())
|
||||
|
||||
# Check the result structure matches OpenAI API format
|
||||
assert isinstance(result, dict)
|
||||
@@ -105,6 +114,33 @@ def test_generate_mm_item_video(video_dataset: RandomMultiModalDataset):
|
||||
os.unlink(temp_path)
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
@pytest.mark.parametrize("writer_opened", [False, True], ids=["open", "write"])
|
||||
def test_generate_synthetic_video_cleans_up_on_error(
|
||||
video_dataset: RandomMultiModalDataset,
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
writer_opened: bool,
|
||||
):
|
||||
"""Release the writer and remove temporary storage when encoding fails."""
|
||||
monkeypatch.setattr(tempfile, "tempdir", str(tmp_path))
|
||||
writer = Mock(spec=cv2.VideoWriter)
|
||||
writer.isOpened.return_value = writer_opened
|
||||
writer.write.side_effect = RuntimeError("Failed to write video frame")
|
||||
monkeypatch.setattr(cv2, "VideoWriter", lambda *args, **kwargs: writer)
|
||||
|
||||
error = (
|
||||
"Failed to write video frame"
|
||||
if writer_opened
|
||||
else "Failed to create video writer"
|
||||
)
|
||||
with pytest.raises(RuntimeError, match=error):
|
||||
video_dataset.generate_synthetic_video(48, 64, 8)
|
||||
|
||||
assert not list(tmp_path.iterdir())
|
||||
writer.release.assert_called_once_with()
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_generate_mm_item_image(video_dataset: RandomMultiModalDataset):
|
||||
"""Test generating multimodal items for image configurations."""
|
||||
|
||||
@@ -19,7 +19,7 @@ class _FakeTokenizer(TokenizerLike):
|
||||
# -- Properties required by TokenizerLike --
|
||||
|
||||
@classmethod
|
||||
def from_pretrained(cls, path_or_repo_id, *a, **kw): # type: ignore[override]
|
||||
def from_pretrained(cls, path_or_repo_id, *a, **kw):
|
||||
return cls()
|
||||
|
||||
@property
|
||||
@@ -80,16 +80,16 @@ class _FakeTokenizer(TokenizerLike):
|
||||
def apply_chat_template(self, messages, **kw): # type: ignore[override]
|
||||
raise NotImplementedError
|
||||
|
||||
def convert_tokens_to_ids(self, tokens): # type: ignore[override]
|
||||
def convert_tokens_to_ids(self, tokens):
|
||||
raise NotImplementedError
|
||||
|
||||
def convert_tokens_to_string(self, tokens: list[str]) -> str:
|
||||
raise NotImplementedError
|
||||
|
||||
def decode(self, ids, skip_special_tokens: bool = False) -> str: # type: ignore[override]
|
||||
def decode(self, ids, skip_special_tokens: bool = False) -> str:
|
||||
raise NotImplementedError
|
||||
|
||||
def convert_ids_to_tokens( # type: ignore[override]
|
||||
def convert_ids_to_tokens(
|
||||
self, ids, skip_special_tokens: bool = False
|
||||
) -> list[str]:
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -9,6 +9,7 @@ serve-side dataset coverage.
|
||||
"""
|
||||
|
||||
import subprocess
|
||||
from argparse import Namespace
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
@@ -289,7 +290,7 @@ def test_to_serve_args_supports_mm_processor_flags() -> None:
|
||||
explicit 0 instead of coercing it to None (which later crashed
|
||||
RandomMultiModalDataset.get_prefix on ``None <= 0``).
|
||||
"""
|
||||
args = SimpleNamespace(
|
||||
args = Namespace(
|
||||
random_input_len=32,
|
||||
random_output_len=1,
|
||||
random_prefix_len=0,
|
||||
@@ -306,7 +307,7 @@ def test_to_serve_args_supports_mm_processor_flags() -> None:
|
||||
|
||||
def test_to_serve_args_keeps_legacy_flag_fallback() -> None:
|
||||
"""Callers that only set the legacy flags still get them mapped."""
|
||||
args = SimpleNamespace(
|
||||
args = Namespace(
|
||||
input_len=256,
|
||||
output_len=8,
|
||||
prefix_len=16,
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
import json
|
||||
import logging
|
||||
from contextlib import nullcontext
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -174,13 +175,18 @@ def test_async_tp_pass_correctness(
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
def test_async_tp_pass_nvfp4_correctness(num_gpus_available: int):
|
||||
if (
|
||||
not current_platform.is_cuda()
|
||||
or not current_platform.is_device_capability_family(100)
|
||||
):
|
||||
pytest.skip("NVFP4 requires Blackwell")
|
||||
if not has_flashinfer():
|
||||
def test_async_tp_pass_nvfp4_correctness(num_gpus_available: int, caplog_mp_spawn):
|
||||
is_blackwell = current_platform.is_cuda() and (
|
||||
current_platform.is_device_capability_family(100)
|
||||
)
|
||||
is_rocm_cdna = False
|
||||
if current_platform.is_rocm():
|
||||
from vllm.platforms.rocm import on_cdna
|
||||
|
||||
is_rocm_cdna = on_cdna()
|
||||
if not (is_blackwell or is_rocm_cdna):
|
||||
pytest.skip("NVFP4 AsyncTP requires Blackwell or ROCm CDNA")
|
||||
if is_blackwell and not has_flashinfer():
|
||||
pytest.skip("FlashInfer is required for the NVFP4 AsyncTP path")
|
||||
|
||||
tp_size = 2
|
||||
@@ -196,8 +202,7 @@ def test_async_tp_pass_nvfp4_correctness(num_gpus_available: int):
|
||||
"8",
|
||||
"--load-format",
|
||||
"dummy",
|
||||
"--linear-backend",
|
||||
"flashinfer_cutlass",
|
||||
*(["--linear-backend", "flashinfer_cutlass"] if is_blackwell else []),
|
||||
"--hf-overrides",
|
||||
json.dumps(NVFP4_HF_OVERRIDES),
|
||||
]
|
||||
@@ -213,6 +218,8 @@ def test_async_tp_pass_nvfp4_correctness(num_gpus_available: int):
|
||||
"sp_min_token_num": 1,
|
||||
},
|
||||
}
|
||||
if is_rocm_cdna:
|
||||
compilation_config["inductor_compile_config"] = {"force_disable_caches": True}
|
||||
|
||||
async_tp_args = [
|
||||
*common_args,
|
||||
@@ -232,10 +239,32 @@ def test_async_tp_pass_nvfp4_correctness(num_gpus_available: int):
|
||||
"mp",
|
||||
]
|
||||
|
||||
compare_two_settings(
|
||||
NVFP4_MODEL_ID,
|
||||
async_tp_args,
|
||||
tp_args,
|
||||
method="generate",
|
||||
force_v1_runner=True,
|
||||
)
|
||||
env = {"VLLM_DISABLE_COMPILE_CACHE": "1"} if is_rocm_cdna else None
|
||||
log_context = caplog_mp_spawn(logging.DEBUG) if is_rocm_cdna else nullcontext()
|
||||
with log_context as logs:
|
||||
compare_two_settings(
|
||||
NVFP4_MODEL_ID,
|
||||
async_tp_args,
|
||||
tp_args,
|
||||
env1=env,
|
||||
env2=env,
|
||||
method="generate",
|
||||
force_v1_runner=True,
|
||||
)
|
||||
if is_rocm_cdna:
|
||||
assert logs is not None
|
||||
matches_by_rank = {
|
||||
rank: [
|
||||
int(count)
|
||||
for line in logs.text.splitlines()
|
||||
if line.startswith(f"(Worker_TP{rank} pid=")
|
||||
for count in FUSION_LOG_PATTERNS["async_tp"].findall(line)
|
||||
]
|
||||
for rank in range(tp_size)
|
||||
}
|
||||
assert all(
|
||||
any(count > 0 for count in matches) for matches in matches_by_rank.values()
|
||||
), (
|
||||
"Expected positive NVFP4 AsyncTP rewrites on both TP workers, "
|
||||
f"found {matches_by_rank}"
|
||||
)
|
||||
|
||||
@@ -374,11 +374,16 @@ def test_tp_sp_generation_prompt_embeds(
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
def test_tp_sp_nvfp4_generation(num_gpus_available: int):
|
||||
if (
|
||||
not current_platform.is_cuda()
|
||||
or not current_platform.is_device_capability_family(100)
|
||||
):
|
||||
pytest.skip("NVFP4 requires Blackwell")
|
||||
is_blackwell = current_platform.is_cuda() and (
|
||||
current_platform.is_device_capability_family(100)
|
||||
)
|
||||
is_rocm_cdna = False
|
||||
if current_platform.is_rocm():
|
||||
from vllm.platforms.rocm import on_cdna
|
||||
|
||||
is_rocm_cdna = on_cdna()
|
||||
if not (is_blackwell or is_rocm_cdna):
|
||||
pytest.skip("NVFP4 sequence parallelism requires Blackwell or ROCm CDNA")
|
||||
|
||||
comparison = _build_sp_args(
|
||||
NVFP4_MODEL_ID,
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from functools import partial
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.models.utils import dummy_hf_overrides
|
||||
from vllm.compilation.counter import compilation_counter
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.compilation import CompilationMode
|
||||
from vllm.config.compilation import CompilationMode, CUDAGraphMode
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
@@ -14,6 +17,77 @@ def test_compile():
|
||||
assert not vllm_config.compilation_config.compile_mm_encoder
|
||||
|
||||
|
||||
@pytest.mark.forked
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Skip if not cuda")
|
||||
@pytest.mark.parametrize(
|
||||
("model", "tokenizer", "config_format", "tokenizer_mode", "model_arch"),
|
||||
[
|
||||
pytest.param(
|
||||
"mistralai/Pixtral-12B-2409",
|
||||
None,
|
||||
"mistral",
|
||||
"mistral",
|
||||
"PixtralForConditionalGeneration",
|
||||
id="pixtral",
|
||||
),
|
||||
pytest.param(
|
||||
"mistralai/Mistral-Small-3.1-24B-Instruct-2503",
|
||||
"mistral-community/pixtral-12b",
|
||||
"hf",
|
||||
"auto",
|
||||
"Mistral3ForConditionalGeneration",
|
||||
id="mistral3-hf",
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("compile_mm_encoder", "expected_models"),
|
||||
[
|
||||
# The dummy model has two compiled text components. Enabling encoder
|
||||
# compilation adds its single vision transformer block.
|
||||
pytest.param(False, 2, id="disabled"),
|
||||
pytest.param(True, 3, id="enabled"),
|
||||
],
|
||||
)
|
||||
def test_pixtral_compilation(
|
||||
vllm_runner,
|
||||
monkeypatch,
|
||||
model: str,
|
||||
tokenizer: str | None,
|
||||
config_format: str,
|
||||
tokenizer_mode: str,
|
||||
model_arch: str,
|
||||
compile_mm_encoder: bool,
|
||||
expected_models: int,
|
||||
):
|
||||
"""Test compilation for both Pixtral vision encoder implementations."""
|
||||
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
monkeypatch.setenv("VLLM_USE_FLASHINFER_SAMPLER", "0")
|
||||
|
||||
with (
|
||||
compilation_counter.expect(num_models_seen=expected_models),
|
||||
vllm_runner(
|
||||
model,
|
||||
tokenizer_name=tokenizer,
|
||||
tokenizer_mode=tokenizer_mode,
|
||||
config_format=config_format,
|
||||
load_format="dummy",
|
||||
hf_overrides=partial(dummy_hf_overrides, model_arch=model_arch),
|
||||
max_model_len=4096,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
gpu_memory_utilization=0.8,
|
||||
attention_backend="FLASH_ATTN",
|
||||
compilation_config={
|
||||
"mode": CompilationMode.VLLM_COMPILE,
|
||||
"compile_mm_encoder": compile_mm_encoder,
|
||||
"cudagraph_mode": CUDAGraphMode.NONE,
|
||||
},
|
||||
) as _,
|
||||
):
|
||||
pass
|
||||
|
||||
|
||||
# forked needed to workaround https://github.com/vllm-project/vllm/issues/21073
|
||||
@pytest.mark.forked
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Skip if not cuda")
|
||||
|
||||
@@ -111,7 +111,9 @@ def test_tp2_ar_rms_fp8_fusions(
|
||||
model_kwargs["max_model_len"] = 1024
|
||||
model_kwargs["kernel_config"] = {"enable_flashinfer_autotune": False}
|
||||
model_kwargs["disable_custom_all_reduce"] = False
|
||||
if current_platform.is_rocm() and "llama-3" in model_name_lower:
|
||||
if current_platform.is_rocm() and (
|
||||
"llama-3" in model_name_lower or "qwen" in model_name_lower
|
||||
):
|
||||
model_kwargs.update(
|
||||
gpu_memory_utilization=0.01,
|
||||
kv_cache_memory_bytes=64 * 1024**2,
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""ROCm MoE coverage for the TP2 GEMM/collective fusion contract."""
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.utils import multi_gpu_test
|
||||
from vllm.config import PassConfig
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from .common import INDUCTOR_GRAPH_PARTITION
|
||||
from .models import TRITON_ATTN, qwen3_a3b
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm AsyncTP contract")
|
||||
@pytest.mark.parametrize("inductor_graph_partition", INDUCTOR_GRAPH_PARTITION[:1])
|
||||
@pytest.mark.parametrize("attn_backend", [TRITON_ATTN])
|
||||
def test_tp2_rocm_moe_async_tp_bf16_fusions(
|
||||
inductor_graph_partition, attn_backend, run_e2e_fusion_test
|
||||
):
|
||||
"""MoE dense projections retain the Qwen SP/AsyncTP rewrite counts."""
|
||||
n_layers = 4
|
||||
run_e2e_fusion_test(
|
||||
qwen3_a3b.model_name,
|
||||
qwen3_a3b.matches(n_layers),
|
||||
dict(
|
||||
hf_overrides=dict(
|
||||
num_hidden_layers=n_layers,
|
||||
hidden_size=512,
|
||||
intermediate_size=1024,
|
||||
moe_intermediate_size=128,
|
||||
num_attention_heads=8,
|
||||
num_key_value_heads=2,
|
||||
head_dim=64,
|
||||
num_experts=8,
|
||||
num_experts_per_tok=2,
|
||||
),
|
||||
load_format="dummy",
|
||||
max_model_len=1024,
|
||||
kv_cache_memory_bytes=64 * 1024**2,
|
||||
gpu_memory_utilization=0.01,
|
||||
max_num_batched_tokens=1024,
|
||||
max_num_seqs=32,
|
||||
),
|
||||
attn_backend,
|
||||
dict(
|
||||
use_inductor_graph_partition=inductor_graph_partition,
|
||||
custom_ops=["-rms_norm"],
|
||||
pass_config=PassConfig(
|
||||
enable_qk_norm_rope_fusion=True,
|
||||
enable_sp=True,
|
||||
fuse_gemm_comms=True,
|
||||
fuse_allreduce_rms=False,
|
||||
sp_min_token_num=512,
|
||||
),
|
||||
),
|
||||
["norm_rope_fusion", "sequence_parallel", "async_tp"],
|
||||
tp_size=2,
|
||||
)
|
||||
@@ -60,10 +60,6 @@ from vllm.utils.deep_gemm import (
|
||||
is_deep_gemm_supported,
|
||||
)
|
||||
|
||||
FP8_DTYPE = current_platform.fp8_dtype()
|
||||
|
||||
RMS_ADD_OP = torch.ops._C.fused_add_rms_norm.default
|
||||
|
||||
# Kernel and group_shape combinations: (kernel, group_shape)
|
||||
# CUDA kernels
|
||||
CUDA_KERNEL_GROUPSHAPE_COMBINATIONS = [
|
||||
|
||||
@@ -33,6 +33,7 @@ from vllm.config import (
|
||||
VllmConfig,
|
||||
set_current_vllm_config,
|
||||
)
|
||||
from vllm.config.cache import CacheDType
|
||||
from vllm.forward_context import get_forward_context, set_forward_context
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
@@ -198,6 +199,7 @@ class TestAttentionAiterFp8StaticQuantPatternModel(
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.fp8_linear.kernel.quant_fp8.use_aiter = True
|
||||
assert self.fp8_linear.input_scale is not None
|
||||
self.fp8_linear.input_scale.fill_(0.125)
|
||||
|
||||
|
||||
@@ -246,7 +248,6 @@ class TestAttentionNvfp4QuantPatternModel(AttentionQuantPatternModel):
|
||||
PATTERN_TEST_MODELS_FP8: list[tuple[str, type]] = []
|
||||
PATTERN_TEST_MODELS_FP4: list[tuple[str, type]] = []
|
||||
HEADS: list[tuple[int, int]] = []
|
||||
SPLIT_ATTENTION: list[bool] = []
|
||||
BACKENDS_FP8: list[AttentionBackendEnum] = []
|
||||
BACKENDS_FP4: list[AttentionBackendEnum] = []
|
||||
|
||||
@@ -307,7 +308,7 @@ def test_attention_quant_pattern(
|
||||
head_size: int,
|
||||
batch_size: int,
|
||||
dtype: torch.dtype,
|
||||
kv_cache_dtype: str,
|
||||
kv_cache_dtype: CacheDType,
|
||||
custom_ops: str,
|
||||
model_name: str,
|
||||
model_class: type[AttentionQuantPatternModel],
|
||||
|
||||
@@ -24,6 +24,7 @@ from vllm.config import (
|
||||
PassConfig,
|
||||
VllmConfig,
|
||||
)
|
||||
from vllm.config.cache import CacheDType
|
||||
from vllm.forward_context import get_forward_context, set_forward_context
|
||||
from vllm.model_executor.layers.attention import MLAAttention
|
||||
from vllm.model_executor.layers.linear import ColumnParallelLinear
|
||||
@@ -254,7 +255,7 @@ def test_mla_rope_kvcache_cat_fusion(
|
||||
block_size: int,
|
||||
is_neox: bool,
|
||||
dtype: torch.dtype,
|
||||
kv_cache_dtype: str,
|
||||
kv_cache_dtype: CacheDType,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
torch.set_default_device("cuda")
|
||||
|
||||
@@ -28,7 +28,7 @@ def test_bad_callable():
|
||||
pass_manager.configure(config)
|
||||
|
||||
with pytest.raises(AssertionError):
|
||||
pass_manager.add(simple_callable) # type: ignore[arg-type]
|
||||
pass_manager.add(simple_callable)
|
||||
|
||||
|
||||
# Pass that inherits from InductorPass
|
||||
|
||||
@@ -39,6 +39,7 @@ from vllm.config import (
|
||||
PassConfig,
|
||||
VllmConfig,
|
||||
)
|
||||
from vllm.config.cache import CacheDType
|
||||
from vllm.forward_context import get_forward_context, set_forward_context
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
@@ -342,7 +343,7 @@ def _run_qk_norm_rope_kvcache_fusion_test(
|
||||
use_shuffle_kv_layout: str,
|
||||
kv_layout: KVCacheLayout,
|
||||
dtype: torch.dtype,
|
||||
kv_cache_dtype: str,
|
||||
kv_cache_dtype: CacheDType,
|
||||
rms_norm_eps: float,
|
||||
custom_op: str,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
@@ -611,7 +612,7 @@ def test_qk_norm_rope_kvcache_fusion(
|
||||
kv_layout: KVCacheLayout,
|
||||
block_size: int,
|
||||
dtype: torch.dtype,
|
||||
kv_cache_dtype: str,
|
||||
kv_cache_dtype: CacheDType,
|
||||
rms_norm_eps: float,
|
||||
custom_op: str,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
@@ -663,7 +664,7 @@ def test_qk_norm_mrope_kvcache_fusion(
|
||||
mrope_interleaved: bool,
|
||||
is_neox: bool,
|
||||
kv_layout: KVCacheLayout,
|
||||
kv_cache_dtype: str,
|
||||
kv_cache_dtype: CacheDType,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
_run_qk_norm_rope_kvcache_fusion_test(
|
||||
|
||||
@@ -30,6 +30,7 @@ from vllm.config import (
|
||||
PassConfig,
|
||||
VllmConfig,
|
||||
)
|
||||
from vllm.config.cache import CacheDType
|
||||
from vllm.config.utils import Range
|
||||
from vllm.forward_context import get_forward_context, set_forward_context
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
@@ -324,7 +325,7 @@ def test_rope_kvcache_fusion(
|
||||
block_size: int,
|
||||
is_neox: bool,
|
||||
dtype: torch.dtype,
|
||||
kv_cache_dtype: str,
|
||||
kv_cache_dtype: CacheDType,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
torch.set_default_device("cuda")
|
||||
@@ -468,7 +469,7 @@ def test_rope_static_qquant_kvcache_fusion(
|
||||
block_size: int,
|
||||
is_neox: bool,
|
||||
dtype: torch.dtype,
|
||||
kv_cache_dtype: str,
|
||||
kv_cache_dtype: CacheDType,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
torch.set_default_device("cuda")
|
||||
|
||||
@@ -60,7 +60,7 @@ class TestSiluMulFp8QuantModel(torch.nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size: int,
|
||||
force_kernel: FP8ScaledMMLinearKernel,
|
||||
force_kernel: type[FP8ScaledMMLinearKernel],
|
||||
dtype: torch.dtype,
|
||||
**kwargs,
|
||||
):
|
||||
@@ -282,7 +282,7 @@ def test_fusion_silu_and_mul_quant(
|
||||
],
|
||||
enable_silu_mul_custom_op: bool,
|
||||
enable_quant_fp8_custom_op: bool,
|
||||
force_kernel: FP8ScaledMMLinearKernel | None,
|
||||
force_kernel: type[FP8ScaledMMLinearKernel] | None,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
if model_class is TestSiluMulNvfp4QuantModel and not is_nvfp4_supported():
|
||||
@@ -330,7 +330,10 @@ def test_fusion_silu_and_mul_quant(
|
||||
passes = [NoOpEliminationPass(config), *fusion_passes, PostCleanupPass(config)]
|
||||
backend = TestBackend(*passes)
|
||||
model = model_class(
|
||||
hidden_size=hidden_size, force_kernel=force_kernel, x=x, dtype=dtype
|
||||
hidden_size=hidden_size,
|
||||
force_kernel=force_kernel,
|
||||
x=x,
|
||||
dtype=dtype,
|
||||
)
|
||||
|
||||
# First dimension dynamic
|
||||
|
||||
@@ -84,7 +84,7 @@ def test_manual_fusion_fp8_static_with_linear(
|
||||
num_tokens: int,
|
||||
hidden_size: int,
|
||||
dtype: torch.dtype,
|
||||
force_kernel: FP8ScaledMMLinearKernel,
|
||||
force_kernel: type[FP8ScaledMMLinearKernel],
|
||||
):
|
||||
"""Test manual fusion with real FP8 linear layer (kFp8StaticTensorSym).
|
||||
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.compilation.compiler_interface import InductorStandaloneAdaptor
|
||||
|
||||
|
||||
@pytest.mark.parametrize("save_format", ["binary", "unpacked"])
|
||||
def test_inductor_standalone_load_uses_current_cache_dir(
|
||||
tmp_path: Path,
|
||||
save_format: str,
|
||||
):
|
||||
old_cache_dir = tmp_path / "old"
|
||||
new_cache_dir = tmp_path / "new"
|
||||
key = "artifact_shape_None_subgraph_0"
|
||||
|
||||
adaptor = InductorStandaloneAdaptor(save_format=save_format)
|
||||
adaptor.initialize_cache(str(new_cache_dir))
|
||||
|
||||
# Simulate a handle persisted before the cache directory was relocated.
|
||||
handle = (key, str(old_cache_dir / key))
|
||||
|
||||
with (
|
||||
patch(
|
||||
"torch._inductor.CompiledArtifact.load",
|
||||
return_value=MagicMock(),
|
||||
) as load_mock,
|
||||
patch(
|
||||
"torch._inductor.compile_fx.graph_returns_tuple",
|
||||
return_value=True,
|
||||
),
|
||||
):
|
||||
adaptor.load(
|
||||
handle=handle,
|
||||
graph=MagicMock(),
|
||||
example_inputs=[],
|
||||
graph_index=0,
|
||||
compile_range=MagicMock(),
|
||||
)
|
||||
|
||||
load_mock.assert_called_once_with(
|
||||
path=str(new_cache_dir / key),
|
||||
format=save_format,
|
||||
)
|
||||
@@ -133,7 +133,7 @@ def test_get_diff_sampling_param_includes_penalties():
|
||||
},
|
||||
)
|
||||
|
||||
diff = ModelConfig.get_diff_sampling_param(fake) # type: ignore[arg-type]
|
||||
diff = ModelConfig.get_diff_sampling_param(fake)
|
||||
|
||||
assert diff["presence_penalty"] == 1.5
|
||||
assert diff["frequency_penalty"] == 0.5
|
||||
@@ -161,7 +161,7 @@ def test_generation_config_penalties_survive_diff_sampling_filter():
|
||||
},
|
||||
)
|
||||
|
||||
diff = ModelConfig.get_diff_sampling_param(fake) # type: ignore[arg-type]
|
||||
diff = ModelConfig.get_diff_sampling_param(fake)
|
||||
|
||||
assert diff["presence_penalty"] == 1.1
|
||||
assert diff["frequency_penalty"] == 0.7
|
||||
|
||||
@@ -8,7 +8,12 @@ import pytest
|
||||
|
||||
from vllm.config.cache import CacheConfig
|
||||
from vllm.config.scheduler import SchedulerConfig
|
||||
from vllm.config.utils import get_hash_factors, hash_factors, normalize_value
|
||||
from vllm.config.utils import (
|
||||
get_attr_docs,
|
||||
get_hash_factors,
|
||||
hash_factors,
|
||||
normalize_value,
|
||||
)
|
||||
|
||||
# Helpers
|
||||
|
||||
@@ -114,7 +119,7 @@ def test_normalize_value_uuid_and_to_json():
|
||||
[
|
||||
(lambda x: x),
|
||||
(type("CallableInstance", (), {"__call__": lambda self: 0}))(),
|
||||
(lambda: (lambda: 0))(), # nested function instance
|
||||
(lambda: lambda: 0)(), # nested function instance
|
||||
],
|
||||
)
|
||||
def test_error_cases(bad):
|
||||
@@ -168,6 +173,53 @@ def test_classes_are_types():
|
||||
assert endswith_fqname(LocalDummy, ".LocalDummy")
|
||||
|
||||
|
||||
def test_ir_config_hash_initializes_provider_and_tracks_implementation(monkeypatch):
|
||||
from types import SimpleNamespace
|
||||
|
||||
from vllm.config.kernel import IrOpPriorityConfig
|
||||
from vllm.ir.op import IrOp
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
op = IrOp.registry["rms_norm"]
|
||||
implementation = SimpleNamespace(uuid=lambda: "first-implementation")
|
||||
|
||||
def import_ir_kernels():
|
||||
monkeypatch.setitem(op.impls, "test_lazy_provider", implementation)
|
||||
|
||||
monkeypatch.setattr(current_platform, "import_ir_kernels", import_ir_kernels)
|
||||
config = IrOpPriorityConfig(rms_norm=["test_lazy_provider"])
|
||||
first_hash = config.compute_hash()
|
||||
assert config.compute_hash() == first_hash
|
||||
|
||||
implementation.uuid = lambda: "changed-implementation"
|
||||
assert config.compute_hash() != first_hash
|
||||
|
||||
|
||||
def test_get_attr_docs_includes_inherited_fields():
|
||||
"""get_attr_docs must collect attribute docstrings inherited from base
|
||||
classes, not just those declared in the class's own body."""
|
||||
|
||||
@dataclass
|
||||
class Base:
|
||||
inherited: object = None
|
||||
"""doc for inherited (base)."""
|
||||
overridden: object = None
|
||||
"""base version of overridden."""
|
||||
|
||||
@dataclass
|
||||
class Derived(Base):
|
||||
own: object = None
|
||||
"""doc for own (derived)."""
|
||||
overridden: object = None
|
||||
"""derived override of overridden."""
|
||||
|
||||
docs = get_attr_docs(Derived)
|
||||
|
||||
assert docs["own"] == "doc for own (derived)."
|
||||
assert docs["inherited"] == "doc for inherited (base)."
|
||||
assert docs["overridden"] == "derived override of overridden."
|
||||
|
||||
|
||||
def test_envs_compile_factors_stable():
|
||||
"""Test that envs.compile_factors() hash is stable across fresh initializations.
|
||||
|
||||
@@ -303,3 +355,10 @@ print(hash_factors(envs.compile_factors()))
|
||||
"HOME relocation changed the compile-cache env hash - a "
|
||||
"location-only derived var is leaking into the key"
|
||||
)
|
||||
|
||||
|
||||
def test_get_attr_docs_rejects_non_class():
|
||||
"""The annotation forbids it, but the runtime guard still protects
|
||||
untyped callers, so keep asserting the documented TypeError."""
|
||||
with pytest.raises(TypeError, match="Given object was not a class."):
|
||||
get_attr_docs(object()) # type: ignore[arg-type]
|
||||
|
||||
@@ -40,18 +40,18 @@ def test_supports_multimodal_inputs(model_id, limit_mm_per_prompt, expected):
|
||||
|
||||
|
||||
def test_mm_encoder_attn_backend_str_conversion():
|
||||
config = MultiModalConfig(mm_encoder_attn_backend="FLASH_ATTN") # type: ignore[arg-type]
|
||||
config = MultiModalConfig(mm_encoder_attn_backend="FLASH_ATTN")
|
||||
assert config.mm_encoder_attn_backend == AttentionBackendEnum.FLASH_ATTN
|
||||
|
||||
|
||||
def test_mm_encoder_attn_backend_invalid():
|
||||
with pytest.raises(ValueError):
|
||||
MultiModalConfig(mm_encoder_attn_backend="not_a_backend") # type: ignore[arg-type]
|
||||
MultiModalConfig(mm_encoder_attn_backend="not_a_backend")
|
||||
|
||||
|
||||
def test_mm_hasher_algorithm_invalid():
|
||||
with pytest.raises(ValueError, match="mm_hasher_algorithm"):
|
||||
MultiModalConfig(mm_hasher_algorithm="md5") # type: ignore[arg-type]
|
||||
MultiModalConfig(mm_hasher_algorithm="md5")
|
||||
|
||||
|
||||
def test_mm_encoder_attn_backend_hash_updates():
|
||||
@@ -240,6 +240,43 @@ def test_merge_mm_processor_kwargs_scoped_precedence():
|
||||
}
|
||||
|
||||
|
||||
def test_merge_mm_processor_kwargs_preserves_scoped_siblings_and_precedence():
|
||||
config = MultiModalConfig(
|
||||
mm_processor_kwargs={
|
||||
"size": {
|
||||
"shortest_edge": 100,
|
||||
"longest_edge": 1000,
|
||||
},
|
||||
"videos_kwargs": {
|
||||
"fps": 2,
|
||||
"size": {"longest_edge": 1200},
|
||||
},
|
||||
},
|
||||
mm_device_do_normalize=False,
|
||||
)
|
||||
|
||||
assert config.merge_mm_processor_kwargs(
|
||||
{
|
||||
"size": {"longest_edge": 1800},
|
||||
"videos_kwargs": {
|
||||
"size": {"shortest_edge": 200},
|
||||
},
|
||||
}
|
||||
) == {
|
||||
"size": {
|
||||
"shortest_edge": 100,
|
||||
"longest_edge": 1800,
|
||||
},
|
||||
"videos_kwargs": {
|
||||
"fps": 2,
|
||||
"size": {
|
||||
"shortest_edge": 200,
|
||||
"longest_edge": 1800,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_merge_mm_processor_kwargs_mapping_vs_replacement():
|
||||
config = MultiModalConfig(
|
||||
mm_processor_kwargs={
|
||||
@@ -523,7 +560,7 @@ def _resolve_mm_processor_device(
|
||||
"""Run the `auto` resolution and report where the processor ended up."""
|
||||
mm_config = MultiModalConfig(
|
||||
mm_processor_kwargs={} if device is None else {"device": device},
|
||||
mm_tensor_ipc=mm_tensor_ipc, # type: ignore[arg-type]
|
||||
mm_tensor_ipc=mm_tensor_ipc,
|
||||
)
|
||||
model_config = MagicMock(spec=ModelConfig)
|
||||
model_config.multimodal_config = mm_config
|
||||
@@ -625,7 +662,7 @@ def _resolve_mm_video_decode_device(
|
||||
the resulting video media IO kwargs."""
|
||||
mm_config = MultiModalConfig(
|
||||
mm_processor_kwargs={} if device is None else {"device": device},
|
||||
mm_tensor_ipc=mm_tensor_ipc, # type: ignore[arg-type]
|
||||
mm_tensor_ipc=mm_tensor_ipc,
|
||||
media_io_kwargs={} if video_kwargs is None else {"video": dict(video_kwargs)},
|
||||
)
|
||||
model_config = MagicMock(spec=ModelConfig)
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for ObservabilityConfig validation."""
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from vllm.config.observability import ObservabilityConfig
|
||||
|
||||
|
||||
def test_custom_histogram_buckets_default_none():
|
||||
assert ObservabilityConfig().custom_histogram_buckets is None
|
||||
|
||||
|
||||
def test_custom_histogram_buckets_valid():
|
||||
config = ObservabilityConfig(
|
||||
custom_histogram_buckets={
|
||||
"request_latency": [0.01, 0.05, 0.1, 0.5],
|
||||
"time_to_first_token": [1, 2, 5],
|
||||
}
|
||||
)
|
||||
assert config.custom_histogram_buckets == {
|
||||
"request_latency": [0.01, 0.05, 0.1, 0.5],
|
||||
"time_to_first_token": [1.0, 2.0, 5.0],
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("buckets", "match"),
|
||||
[
|
||||
({"bogus": [1.0, 2.0]}, "unknown bucket family 'bogus'"),
|
||||
({"request_latency": []}, "must not be empty"),
|
||||
({"request_latency": [0.0, 1.0]}, "must be finite and greater than 0"),
|
||||
({"request_latency": [-1.0, 1.0]}, "must be finite and greater than 0"),
|
||||
(
|
||||
{"request_latency": [1.0, float("inf")]},
|
||||
"must be finite and greater than 0",
|
||||
),
|
||||
({"request_latency": [float("nan")]}, "must be finite and greater than 0"),
|
||||
({"request_latency": [1.0, 1.0]}, "must be strictly increasing"),
|
||||
({"request_latency": [2.0, 1.0]}, "must be strictly increasing"),
|
||||
({"request_latency": [True, 2.0]}, "must be a number, not a boolean"),
|
||||
({"request_latency": [False]}, "must be a number, not a boolean"),
|
||||
(
|
||||
{"request_latency": [1.0, 2.0], "request_tokens": [16, 8]},
|
||||
r"request_tokens.*must be strictly increasing",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_custom_histogram_buckets_rejects_invalid(buckets, match):
|
||||
"""Each validation rule must name the field, the value, and the rule."""
|
||||
with pytest.raises(ValidationError, match=match):
|
||||
ObservabilityConfig(custom_histogram_buckets=buckets)
|
||||
+1
-6
@@ -317,11 +317,6 @@ def example_system_message() -> str:
|
||||
return f.read()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def example_long_prompts() -> list[str]:
|
||||
return [prompt for filename in _LONG_PROMPTS for prompt in _read_prompts(filename)]
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def image_assets() -> ImageTestAssets:
|
||||
return IMAGE_ASSETS
|
||||
@@ -1168,7 +1163,7 @@ class VllmRunner:
|
||||
output_logprobs = sample.logprobs
|
||||
if include_prompt_token_ids:
|
||||
outputs.append(
|
||||
( # type: ignore[arg-type]
|
||||
(
|
||||
output_ids,
|
||||
output_str,
|
||||
output_logprobs,
|
||||
|
||||
@@ -205,6 +205,42 @@ def test_custom_allreduce_filters_dtype(
|
||||
assert communicator.should_custom_ar(torch.empty(16, dtype=dtype)) is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("batch_invariant", [False, True])
|
||||
def test_custom_allreduce_size_gate_ignored_under_batch_invariance(
|
||||
batch_invariant: bool,
|
||||
) -> None:
|
||||
"""Batch invariance must not switch backends based on the input size."""
|
||||
communicator = car.CustomAllreduce.__new__(car.CustomAllreduce)
|
||||
communicator.disabled = False
|
||||
communicator.world_size = 2
|
||||
communicator.max_size = 1024
|
||||
communicator.batch_invariant = batch_invariant
|
||||
communicator._ptr = 0
|
||||
|
||||
oversized = torch.empty(1024, dtype=torch.float16)
|
||||
assert communicator.should_custom_ar(oversized) is batch_invariant
|
||||
|
||||
|
||||
@pytest.mark.parametrize("batch_invariant", [False, True])
|
||||
def test_custom_reduce_scatter_disabled_under_batch_invariance(
|
||||
monkeypatch, batch_invariant: bool
|
||||
) -> None:
|
||||
"""Reduce-scatter stays on one backend under batch invariance."""
|
||||
monkeypatch.setattr(car.current_platform, "is_cuda", lambda: True)
|
||||
communicator = car.CustomAllreduce.__new__(car.CustomAllreduce)
|
||||
communicator.disabled = False
|
||||
communicator.world_size = 2
|
||||
communicator.fully_connected = True
|
||||
communicator.mnnvl_only = False
|
||||
communicator.mnnvl_multicast_ptr = 0
|
||||
communicator.max_reduce_scatter_size = 1024
|
||||
communicator.max_mnnvl_reduce_scatter_size = 1024
|
||||
communicator.batch_invariant = batch_invariant
|
||||
|
||||
in_range = torch.empty(64, dtype=torch.float16)
|
||||
assert communicator.should_custom_reduce_scatter(in_range) is not batch_invariant
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("major", "local_multicast", "expected"),
|
||||
[
|
||||
@@ -622,6 +658,60 @@ def eager_allreduce(
|
||||
torch.testing.assert_close(out, inp * (tp_size**num_communication))
|
||||
|
||||
|
||||
@ray.remote(num_gpus=1, max_calls=1)
|
||||
def chunked_allreduce(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tp_size,
|
||||
pp_size,
|
||||
rank,
|
||||
distributed_init_port,
|
||||
):
|
||||
"""Inputs above max_size are reduced in chunks and match the fixed-order
|
||||
reference bitwise, both eagerly and inside a CUDA graph."""
|
||||
with monkeypatch.context() as m:
|
||||
m.delenv("CUDA_VISIBLE_DEVICES", raising=False)
|
||||
m.delenv("HIP_VISIBLE_DEVICES", raising=False)
|
||||
m.setenv("VLLM_CUSTOM_ALLREDUCE_ALGO", "1stage")
|
||||
device = torch.device(f"cuda:{rank}")
|
||||
torch.accelerator.set_device_index(device)
|
||||
init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port)
|
||||
ensure_model_parallel_initialized(tp_size, pp_size)
|
||||
group = get_tp_group().device_group
|
||||
fa = get_tp_group().device_communicator.ca_comm
|
||||
|
||||
# Two full chunks plus a partial one.
|
||||
dtype = torch.bfloat16
|
||||
chunk_numel = fa.max_size // dtype.itemsize
|
||||
inp = torch.randn(2 * chunk_numel + 4096, device=device).to(dtype)
|
||||
gathered = [torch.empty_like(inp) for _ in range(tp_size)]
|
||||
dist.all_gather(gathered, inp, group=group)
|
||||
ref = gathered[0].float()
|
||||
for peer in gathered[1:]:
|
||||
ref = ref + peer.float()
|
||||
ref = ref.to(torch.bfloat16)
|
||||
|
||||
out = fa.all_reduce(inp, registered=False)
|
||||
assert torch.equal(out, ref)
|
||||
|
||||
# Weak-contiguous but not C-contiguous: a transposed matrix.
|
||||
inp_t, ref_t = inp.view(-1, 4096).t(), ref.view(-1, 4096).t()
|
||||
assert torch.equal(fa.all_reduce(inp_t, registered=False), ref_t)
|
||||
|
||||
with fa.capture():
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
out = fa.all_reduce(inp, registered=True)
|
||||
graph.replay()
|
||||
assert torch.equal(out, ref)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tp_size", [2, 4])
|
||||
def test_custom_allreduce_chunked(monkeypatch: pytest.MonkeyPatch, tp_size):
|
||||
if tp_size > torch.accelerator.device_count():
|
||||
pytest.skip("Not enough GPUs to run the test.")
|
||||
multi_process_parallel(monkeypatch, tp_size, 1, chunked_allreduce)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tp_size", [2])
|
||||
@pytest.mark.parametrize("pipeline_parallel_size", [1, 2])
|
||||
@pytest.mark.parametrize("test_target", [eager_allreduce, graph_allreduce])
|
||||
|
||||
@@ -39,10 +39,10 @@ from vllm.distributed.parallel_state import (
|
||||
initialize_model_parallel,
|
||||
)
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.models.deepseek_v41.common.engram import EngramLayout
|
||||
from vllm.models.deepseek_v41.nvidia import engram as engram_ops
|
||||
from vllm.models.deepseek_v41.nvidia.engram import (
|
||||
from vllm.models.deepseek_v41.common import engram as engram_ops
|
||||
from vllm.models.deepseek_v41.common.engram import (
|
||||
Engram,
|
||||
EngramLayout,
|
||||
ParallelEngramEmbedding,
|
||||
engram_head_shard_rank,
|
||||
gather_engram_hashes,
|
||||
@@ -507,20 +507,22 @@ def _check_dummy_hash_model_forward(
|
||||
assert torch.all(hashes == engram_ops.DEAD_ID)
|
||||
output = engram.embed(hashes[:, 0])
|
||||
# Trailing None is previous_aux; this stub captures no aux states.
|
||||
return output, None, None, None, None, None
|
||||
return output, hidden, hidden, hidden, hidden, None
|
||||
|
||||
model = SimpleNamespace(
|
||||
use_mega_moe=False,
|
||||
use_sequence_parallel=False,
|
||||
fuse_mhc_all_reduce=False,
|
||||
engram_hash=state,
|
||||
engram_swa_prefix="swa",
|
||||
engram_dp_shared_memory=dp_shared_memory,
|
||||
layers=[Decoder(engram=engram)],
|
||||
start_layer=0,
|
||||
end_layer=1,
|
||||
aux_hidden_state_layers=(),
|
||||
)
|
||||
model = model_ops.DeepseekV4Model.__new__(model_ops.DeepseekV4Model)
|
||||
torch.nn.Module.__init__(model)
|
||||
model.use_mega_moe = False
|
||||
model.use_sequence_parallel = False
|
||||
model.fuse_mhc_all_reduce = False
|
||||
model.engram_hash = state
|
||||
model.engram_swa_prefix = "swa"
|
||||
model.engram_dp_shared_memory = dp_shared_memory
|
||||
model.layers = [Decoder(engram=engram)]
|
||||
model.start_layer = 0
|
||||
model.end_layer = 1
|
||||
model.decoder_replay_start = model.end_layer
|
||||
model.decoder_replay_layers = None
|
||||
model.aux_hidden_state_layers = ()
|
||||
metadata = (
|
||||
None
|
||||
if dp_rank == 1
|
||||
@@ -550,8 +552,7 @@ def _check_dummy_hash_model_forward(
|
||||
dtype=torch.int32,
|
||||
),
|
||||
):
|
||||
output = model_ops.DeepseekV4Model.forward(
|
||||
model,
|
||||
output = model(
|
||||
torch.arange(tokens, device="cuda"),
|
||||
torch.arange(tokens, device="cuda"),
|
||||
intermediate_tensors=None,
|
||||
@@ -685,11 +686,30 @@ def test_engram_dp_shared_memory_runtime_requirements(
|
||||
)
|
||||
|
||||
|
||||
def test_engram_tables_too_large_for_shm_are_not_shared(monkeypatch):
|
||||
"""A /dev/shm smaller than the tables must fall back instead of failing startup."""
|
||||
@pytest.mark.parametrize("query_fails", [False, True])
|
||||
def test_engram_tables_too_large_for_shm_are_not_shared(monkeypatch, query_fails):
|
||||
"""A /dev/shm that is too small or cannot be queried must fall back, and the
|
||||
leader must still broadcast that decision so its peers do not hang."""
|
||||
broadcasts = []
|
||||
|
||||
def broadcast(obj):
|
||||
broadcasts.append(obj)
|
||||
return obj
|
||||
|
||||
def query(*args, **kwargs):
|
||||
raise OSError("statvfs failed")
|
||||
|
||||
monkeypatch.setattr(engram_ops, "get_engram_dp_size", lambda: 2)
|
||||
monkeypatch.setattr(
|
||||
engram_ops,
|
||||
"get_engram_dp_group",
|
||||
lambda: SimpleNamespace(rank_in_group=0, broadcast_object=broadcast),
|
||||
)
|
||||
if query_fails:
|
||||
monkeypatch.setattr(engram_ops, "check_shm_free_space", query)
|
||||
layout = SimpleNamespace(num_embeddings=(1 << 50,), head_dim=DIM)
|
||||
assert not engram_ops.can_share_engram_tables(layout)
|
||||
assert len(broadcasts) == 1 and broadcasts[0] is not None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
||||
@@ -8,6 +8,7 @@ Run: pytest tests/distributed/test_mnnvl_alltoall.py -v
|
||||
|
||||
import os
|
||||
import traceback
|
||||
from contextlib import contextmanager
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -148,16 +149,17 @@ def _init_dp_environment(world_size, rank, port, dp_size, dp_port):
|
||||
ensure_model_parallel_initialized(1, 1)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _make_forward_context(rank, world_size, num_tokens_per_rank):
|
||||
"""Create a forward context with mock DP metadata for AgRs tests.
|
||||
"""Keep the vLLM config and mock-DP forward context active together.
|
||||
|
||||
Returns a context manager suitable for ``with`` statements.
|
||||
The real DPMetadata (with sp_local_sizes etc.) is created internally
|
||||
by set_forward_context from num_tokens_across_dp; the attn_metadata
|
||||
placeholder just satisfies the "attn_metadata is not None" guard.
|
||||
"""
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.config.parallel import ParallelConfig
|
||||
from vllm.config.vllm import VllmConfig
|
||||
from vllm.forward_context import set_forward_context
|
||||
|
||||
class _AttnMeta:
|
||||
@@ -173,14 +175,18 @@ def _make_forward_context(rank, world_size, num_tokens_per_rank):
|
||||
is_moe_model=True,
|
||||
data_parallel_rank=rank,
|
||||
)
|
||||
return set_forward_context(
|
||||
_AttnMeta(),
|
||||
vllm_config,
|
||||
num_tokens=num_tokens_per_rank,
|
||||
num_tokens_across_dp=torch.tensor(
|
||||
[num_tokens_per_rank] * world_size, dtype=torch.int
|
||||
with (
|
||||
set_current_vllm_config(vllm_config),
|
||||
set_forward_context(
|
||||
_AttnMeta(),
|
||||
vllm_config,
|
||||
num_tokens=num_tokens_per_rank,
|
||||
num_tokens_across_dp=torch.tensor(
|
||||
[num_tokens_per_rank] * world_size, dtype=torch.int
|
||||
),
|
||||
),
|
||||
)
|
||||
):
|
||||
yield
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -108,10 +108,10 @@ class TestSingleWriterShmRingBuffer(unittest.TestCase):
|
||||
# Clear the buffer
|
||||
self.ring_buffer.clear()
|
||||
|
||||
# Check that metadata is empty and IDs reset
|
||||
# Check that metadata is empty and IDs keep increasing
|
||||
self.assertEqual(len(self.ring_buffer.metadata), 0)
|
||||
self.assertEqual(self.ring_buffer.monotonic_id_start, 0)
|
||||
self.assertEqual(self.ring_buffer.monotonic_id_end, 0)
|
||||
self.assertEqual(self.ring_buffer.monotonic_id_start, 3)
|
||||
self.assertEqual(self.ring_buffer.monotonic_id_end, 3)
|
||||
self.assertEqual(self.ring_buffer.data_buffer_start, 0)
|
||||
self.assertEqual(self.ring_buffer.data_buffer_end, 0)
|
||||
|
||||
|
||||
@@ -227,10 +227,68 @@ class TestSingleWriterShmObjectStorage(unittest.TestCase):
|
||||
self.assertEqual(len(self.storage.id_index), 0)
|
||||
self.assertEqual(len(self.storage.ring_buffer.metadata), 0)
|
||||
|
||||
# Verify that new items can be added after clearing
|
||||
# Verify that new items can be added after clearing. IDs keep
|
||||
# increasing so handles issued before the clear cannot alias new data.
|
||||
address, monotonic_id = self.storage.put("new_item", "new_value")
|
||||
self.assertIn("new_item", self.storage.key_index)
|
||||
self.assertEqual((address, monotonic_id), (0, 0))
|
||||
self.assertEqual((address, monotonic_id), (0, 5))
|
||||
|
||||
def _open_reader(self) -> SingleWriterShmObjectStorage:
|
||||
reader = SingleWriterShmObjectStorage.create_from_handle(self.storage.handle())
|
||||
self.addCleanup(reader.close)
|
||||
return reader
|
||||
|
||||
def test_reader_get_and_touch_each_require_valid_signature(self):
|
||||
address, monotonic_id = self.storage.put("key", "value")
|
||||
signature = self.storage.get_signature("key")
|
||||
reader = self._open_reader()
|
||||
|
||||
reader.touch("key", address, monotonic_id, signature)
|
||||
self.assertEqual(reader.get(address, monotonic_id, signature, "key"), "value")
|
||||
|
||||
flipped = [signature[0] ^ 1, *signature[1:]]
|
||||
for bad in (None, flipped, [signature], signature[:-1], [256] * 32):
|
||||
with self.assertRaisesRegex(ValueError, "SHM handle signature"):
|
||||
reader.get(address, monotonic_id, bad, "key")
|
||||
with self.assertRaisesRegex(ValueError, "SHM handle signature"):
|
||||
reader.touch("key", address, monotonic_id, bad)
|
||||
|
||||
def test_signature_is_bound_to_key_and_address(self):
|
||||
address, monotonic_id = self.storage.put("key", "value")
|
||||
signature = self.storage.get_signature("key")
|
||||
reader = self._open_reader()
|
||||
|
||||
with self.assertRaisesRegex(ValueError, "SHM handle signature"):
|
||||
reader.get(address, monotonic_id, signature, "other_key")
|
||||
with self.assertRaisesRegex(ValueError, "SHM handle signature"):
|
||||
reader.get(address + 1, monotonic_id, signature, "key")
|
||||
|
||||
def test_writer_rejects_handle_no_longer_in_key_index(self):
|
||||
address, monotonic_id = self.storage.put("key", "value")
|
||||
signature = self.storage.get_signature("key")
|
||||
self.storage.verify_signature("key", address, monotonic_id, signature)
|
||||
|
||||
self.storage.clear()
|
||||
|
||||
with self.assertRaisesRegex(ValueError, "SHM handle signature"):
|
||||
self.storage.verify_signature("key", address, monotonic_id, signature)
|
||||
|
||||
def test_handle_issued_before_clear_reads_own_data_until_reused(self):
|
||||
address, monotonic_id = self.storage.put("key", "value")
|
||||
signature = self.storage.get_signature("key")
|
||||
reader = self._open_reader()
|
||||
|
||||
self.storage.clear()
|
||||
|
||||
# Requests still draining after a clear keep reading their own data.
|
||||
self.assertEqual(reader.get(address, monotonic_id, signature, "key"), "value")
|
||||
|
||||
self.storage.put("key", "new_value")
|
||||
|
||||
with self.assertRaisesRegex(ValueError, "modified or is invalid"):
|
||||
reader.get(address, monotonic_id, signature, "key")
|
||||
with self.assertRaisesRegex(ValueError, "modified or is invalid"):
|
||||
reader.touch("key", address, monotonic_id, signature)
|
||||
|
||||
|
||||
# Reader process function
|
||||
@@ -242,11 +300,11 @@ def reader_process(process_id, storage_handle, items_to_read):
|
||||
|
||||
errors = []
|
||||
|
||||
for key, original_value, address, monotonic_id in items_to_read:
|
||||
for key, original_value, address, monotonic_id, signature in items_to_read:
|
||||
time.sleep(random.random() / 100)
|
||||
try:
|
||||
# Read data from shared memory
|
||||
retrieved_value = reader_storage.get(address, monotonic_id)
|
||||
retrieved_value = reader_storage.get(address, monotonic_id, signature, key)
|
||||
|
||||
# Verify data integrity
|
||||
assert retrieved_value == original_value
|
||||
@@ -289,7 +347,8 @@ def run_multiprocess_example():
|
||||
for key, value in test_data:
|
||||
print(f"Storing {key}: {value}")
|
||||
address, monotonic_id = storage.put(key, value)
|
||||
stored_items.append((key, value, address, monotonic_id))
|
||||
signature = storage.get_signature(key)
|
||||
stored_items.append((key, value, address, monotonic_id, signature))
|
||||
print(f" -> Stored at address {address}, ID {monotonic_id}")
|
||||
|
||||
print("\n--- Retrieving Data ---")
|
||||
|
||||
@@ -6,10 +6,14 @@ Unit tests for engine classes (parsing, validation, registry).
|
||||
Integration tests for NCCL and IPC weight transfer between processes using Ray.
|
||||
"""
|
||||
|
||||
import builtins
|
||||
import pickle
|
||||
import runpy
|
||||
import threading
|
||||
import time
|
||||
from contextlib import nullcontext
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pybase64 as base64
|
||||
@@ -64,6 +68,7 @@ from vllm.distributed.weight_transfer.sparse_nccl_engine import (
|
||||
SparseWeightPatch,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.nccl import _nccl_has_no_cache
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
|
||||
|
||||
@@ -297,6 +302,36 @@ class TestEngineRegistry:
|
||||
)
|
||||
assert isinstance(engine, SparseNCCLWeightTransferEngine)
|
||||
|
||||
def test_modelexpress_registration_is_native_and_lazy(self, monkeypatch):
|
||||
from vllm.distributed.weight_transfer import factory
|
||||
|
||||
imported = []
|
||||
module = MagicMock()
|
||||
module.ModelExpressWeightTransferEngine.__name__ = (
|
||||
"ModelExpressWeightTransferEngine"
|
||||
)
|
||||
|
||||
def import_module(name):
|
||||
imported.append(name)
|
||||
return module
|
||||
|
||||
monkeypatch.setattr(factory.importlib, "import_module", import_module)
|
||||
config = WeightTransferConfig(backend="modelexpress")
|
||||
vllm_config = create_mock_vllm_config()
|
||||
device = torch.device("cpu")
|
||||
model = MagicMock(spec=torch.nn.Module)
|
||||
|
||||
assert imported == []
|
||||
engine = WeightTransferEngineFactory.create_engine(
|
||||
config, vllm_config, device, model
|
||||
)
|
||||
|
||||
assert imported == ["vllm.distributed.weight_transfer.modelexpress_engine"]
|
||||
module.ModelExpressWeightTransferEngine.assert_called_once_with(
|
||||
config, vllm_config, device, model
|
||||
)
|
||||
assert engine is module.ModelExpressWeightTransferEngine.return_value
|
||||
|
||||
def test_create_engine_invalid_backend(self):
|
||||
config = WeightTransferConfig(backend="invalid")
|
||||
with pytest.raises(ValueError, match="Invalid weight transfer backend"):
|
||||
@@ -399,13 +434,15 @@ def trainer_broadcast_tensor(
|
||||
return True
|
||||
|
||||
|
||||
@ray.remote(num_gpus=1)
|
||||
# max_calls=1: a batch-invariant run leaves NCCL pins in the worker process.
|
||||
@ray.remote(num_gpus=1, max_calls=1)
|
||||
def inference_receive_tensor(
|
||||
master_address: str,
|
||||
master_port: int,
|
||||
world_size: int,
|
||||
tensor_shape: list[int],
|
||||
tensor_dtype: str,
|
||||
batch_invariant: bool = False,
|
||||
) -> dict:
|
||||
"""Inference task that receives tensor via NCCLWeightTransferEngine."""
|
||||
import contextlib
|
||||
@@ -414,6 +451,21 @@ def inference_receive_tensor(
|
||||
import torch
|
||||
|
||||
_set_ray_assigned_device()
|
||||
if batch_invariant:
|
||||
from vllm.model_executor.determinism.batch_invariant import (
|
||||
override_envs_for_invariance,
|
||||
)
|
||||
|
||||
override_envs_for_invariance()
|
||||
# Like vLLM's own groups, a pinned communicator makes NCCL read the
|
||||
# pins before the transfer group exists.
|
||||
torch.distributed.init_process_group(
|
||||
"nccl",
|
||||
init_method=f"tcp://127.0.0.1:{get_open_port()}",
|
||||
rank=0,
|
||||
world_size=1,
|
||||
)
|
||||
torch.distributed.all_reduce(torch.ones(1, device="cuda"))
|
||||
|
||||
from vllm.config.parallel import ParallelConfig
|
||||
from vllm.config.weight_transfer import WeightTransferConfig
|
||||
@@ -500,11 +552,25 @@ def inference_receive_tensor(
|
||||
torch.accelerator.device_count() < 2,
|
||||
reason="Need at least 2 GPUs to run NCCL weight transfer test.",
|
||||
)
|
||||
def test_nccl_weight_transfer_between_processes():
|
||||
@pytest.mark.parametrize(
|
||||
"batch_invariant",
|
||||
[
|
||||
pytest.param(False, id="default"),
|
||||
pytest.param(
|
||||
True,
|
||||
id="batch-invariant-worker",
|
||||
marks=pytest.mark.skipif(
|
||||
not _nccl_has_no_cache(), reason="Needs CUDA NCCL >= 2.29.7."
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_nccl_weight_transfer_between_processes(batch_invariant):
|
||||
"""Test NCCL weight transfer from trainer to inference process using Ray.
|
||||
|
||||
This test verifies that the NCCLWeightTransferEngine can receive
|
||||
tensors broadcast by a trainer process via NCCL.
|
||||
tensors broadcast by a trainer process via NCCL, including when the
|
||||
worker runs with batch-invariance NCCL pins the trainer does not have.
|
||||
"""
|
||||
_init_ray_for_weight_transfer()
|
||||
|
||||
@@ -516,13 +582,19 @@ def test_nccl_weight_transfer_between_processes():
|
||||
tensor_dtype = "float32"
|
||||
|
||||
inference_future = inference_receive_tensor.remote(
|
||||
master_address, master_port, world_size, tensor_shape, tensor_dtype
|
||||
master_address,
|
||||
master_port,
|
||||
world_size,
|
||||
tensor_shape,
|
||||
tensor_dtype,
|
||||
batch_invariant,
|
||||
)
|
||||
trainer_future = trainer_broadcast_tensor.remote(
|
||||
master_address, master_port, world_size, tensor_shape, tensor_dtype
|
||||
)
|
||||
|
||||
trainer_result, result = ray.get([trainer_future, inference_future])
|
||||
# A mismatched NCCL config deadlocks instead of failing.
|
||||
trainer_result, result = ray.get([trainer_future, inference_future], timeout=300)
|
||||
|
||||
assert trainer_result, "Trainer should complete successfully"
|
||||
assert result["success"], (
|
||||
@@ -1976,3 +2048,322 @@ def test_sparse_nccl_trainer_non_sender_skips_client():
|
||||
engine.send_weights([_sparse_patch()])
|
||||
engine.send_weight_chunk([_sparse_patch()])
|
||||
assert client.order == []
|
||||
|
||||
|
||||
# --- Optional ModelExpress Client Lifecycle ---
|
||||
|
||||
|
||||
def _import_modelexpress_shim():
|
||||
path = (
|
||||
Path(__file__).parents[2]
|
||||
/ "vllm/distributed/weight_transfer/modelexpress_engine.py"
|
||||
)
|
||||
runpy.run_path(str(path), run_name="_test_modelexpress_shim")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"missing_module",
|
||||
[
|
||||
"modelexpress",
|
||||
"modelexpress_rl",
|
||||
"modelexpress_rl.inference",
|
||||
"modelexpress_rl.inference.engines",
|
||||
"modelexpress_rl.inference.engines.vllm",
|
||||
"modelexpress_rl.inference.engines.vllm.weight_transfer_engine",
|
||||
],
|
||||
)
|
||||
def test_modelexpress_missing_backend_explains_source_install(
|
||||
monkeypatch, missing_module
|
||||
):
|
||||
original_import = builtins.__import__
|
||||
error = ModuleNotFoundError(name=missing_module)
|
||||
|
||||
def import_without_backend(name, *args, **kwargs):
|
||||
if name == "modelexpress":
|
||||
raise error
|
||||
return original_import(name, *args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(builtins, "__import__", import_without_backend)
|
||||
with pytest.raises(ImportError, match="modelexpress_client/python") as exc_info:
|
||||
_import_modelexpress_shim()
|
||||
assert exc_info.value.__cause__ is error
|
||||
assert "uv pip install" in str(exc_info.value)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("missing_module", ["grpc", "modelexpress.internal_dependency"])
|
||||
def test_modelexpress_preserves_transitive_import_errors(monkeypatch, missing_module):
|
||||
original_import = builtins.__import__
|
||||
error = ModuleNotFoundError(name=missing_module)
|
||||
|
||||
def import_without_dependency(name, *args, **kwargs):
|
||||
if name == "modelexpress":
|
||||
raise error
|
||||
return original_import(name, *args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(builtins, "__import__", import_without_dependency)
|
||||
with pytest.raises(ModuleNotFoundError) as exc_info:
|
||||
_import_modelexpress_shim()
|
||||
assert exc_info.value is error
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mx_client(monkeypatch):
|
||||
client_module = pytest.importorskip("modelexpress_rl.inference.client")
|
||||
client = MagicMock()
|
||||
monkeypatch.setattr(
|
||||
client_module.ModelExpressGeneratorClient,
|
||||
"initialize",
|
||||
MagicMock(return_value=client),
|
||||
)
|
||||
monkeypatch.setattr(torch.accelerator, "synchronize", MagicMock())
|
||||
return client
|
||||
|
||||
|
||||
def _make_modelexpress_engine():
|
||||
return WeightTransferEngineFactory.create_engine(
|
||||
WeightTransferConfig(backend="modelexpress"),
|
||||
SimpleNamespace(
|
||||
parallel_config=SimpleNamespace(),
|
||||
model_config=SimpleNamespace(model="test/model"),
|
||||
),
|
||||
torch.device("cpu"),
|
||||
torch.nn.Linear(2, 2),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mx_worker(mx_client, monkeypatch):
|
||||
from vllm.v1.worker import gpu_worker
|
||||
|
||||
monkeypatch.setattr(gpu_worker, "set_current_vllm_config", lambda _: nullcontext())
|
||||
worker = object.__new__(gpu_worker.Worker)
|
||||
worker.weight_transfer_engine = _make_modelexpress_engine()
|
||||
worker.vllm_config = worker.weight_transfer_engine.vllm_config
|
||||
worker._weight_update_active = False
|
||||
worker._weight_update_is_draft = False
|
||||
worker.model_runner = MagicMock()
|
||||
return worker
|
||||
|
||||
|
||||
def test_modelexpress_worker_rejects_updates_before_initialization(
|
||||
mx_worker, mx_client
|
||||
):
|
||||
with pytest.raises(RuntimeError, match="not initialized"):
|
||||
mx_worker.start_weight_update()
|
||||
with pytest.raises(RuntimeError, match="start_weight_update must be called"):
|
||||
mx_worker.update_weights({"version_id": "version-a"})
|
||||
with pytest.raises(RuntimeError, match="without a matching"):
|
||||
mx_worker.finish_weight_update()
|
||||
mx_client.stage_weight.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"second_update, error_type",
|
||||
[
|
||||
({"version_id": "version-a"}, RuntimeError),
|
||||
({"version_id": "version-b"}, RuntimeError),
|
||||
({"version_id": ""}, ValueError),
|
||||
({"version_id": None}, ValueError),
|
||||
({}, ValueError),
|
||||
],
|
||||
)
|
||||
def test_modelexpress_worker_recovers_after_rejected_update(
|
||||
mx_worker, mx_client, second_update, error_type
|
||||
):
|
||||
"""A rejected payload must release A so a fresh session can actually install B."""
|
||||
mx_worker.weight_transfer_engine.init_transfer_engine(
|
||||
mx_worker.weight_transfer_engine.parse_init_info({})
|
||||
)
|
||||
staged_a = SimpleNamespace(version_id="version-a", metrics={}, release=MagicMock())
|
||||
staged_b = SimpleNamespace(version_id="version-b", metrics={}, release=MagicMock())
|
||||
mx_client.stage_weight.side_effect = [staged_a, staged_b]
|
||||
mx_client.apply_weight.return_value = {}
|
||||
mx_worker.start_weight_update()
|
||||
mx_worker.update_weights({"version_id": "version-a"})
|
||||
|
||||
with pytest.raises(error_type):
|
||||
mx_worker.update_weights(second_update)
|
||||
staged_a.release.assert_called_once_with()
|
||||
mx_client.apply_weight.assert_called_once_with(staged_a)
|
||||
|
||||
mx_worker.start_weight_update()
|
||||
mx_worker.update_weights({"version_id": "version-b"})
|
||||
mx_worker.finish_weight_update()
|
||||
|
||||
assert [
|
||||
call.kwargs["version"].version_id
|
||||
for call in mx_client.stage_weight.call_args_list
|
||||
] == ["version-a", "version-b"]
|
||||
mx_client.apply_weight.assert_called_with(staged_b)
|
||||
staged_b.release.assert_called_once_with()
|
||||
|
||||
|
||||
def test_modelexpress_worker_can_retry_failed_finish(mx_worker, mx_client):
|
||||
mx_worker.weight_transfer_engine.init_transfer_engine(
|
||||
mx_worker.weight_transfer_engine.parse_init_info({})
|
||||
)
|
||||
staged = mx_client.stage_weight.return_value
|
||||
staged.release.side_effect = [RuntimeError("release failed"), None]
|
||||
mx_worker.start_weight_update()
|
||||
mx_worker.update_weights({"version_id": "version-a"})
|
||||
|
||||
with pytest.raises(RuntimeError, match="release failed"):
|
||||
mx_worker.finish_weight_update()
|
||||
mx_worker.finish_weight_update()
|
||||
|
||||
assert staged.release.call_count == 2
|
||||
assert not mx_worker._weight_update_active
|
||||
mx_worker.start_weight_update()
|
||||
|
||||
|
||||
def test_native_engine_applies_and_releases_exact_versions(mx_client):
|
||||
from modelexpress_rl.inference.engines.vllm import (
|
||||
weight_transfer_engine as mx_engine,
|
||||
)
|
||||
|
||||
from vllm.distributed.weight_transfer import modelexpress_engine
|
||||
|
||||
engine = _make_modelexpress_engine()
|
||||
assert type(engine) is mx_engine.ModelExpressWeightTransferEngine
|
||||
assert (
|
||||
modelexpress_engine.ModelExpressWeightTransferInitInfo
|
||||
is mx_engine.ModelExpressWeightTransferInitInfo
|
||||
)
|
||||
assert (
|
||||
modelexpress_engine.ModelExpressWeightTransferUpdateInfo
|
||||
is mx_engine.ModelExpressWeightTransferUpdateInfo
|
||||
)
|
||||
assert not engine.supports_draft_weight_update
|
||||
engine.init_transfer_engine(engine.parse_init_info({}))
|
||||
|
||||
for version_id in ("version-a", "version-b"):
|
||||
staged = SimpleNamespace(version_id=version_id, metrics={}, release=MagicMock())
|
||||
mx_client.stage_weight.return_value = staged
|
||||
mx_client.apply_weight.return_value = {}
|
||||
engine.start_weight_update()
|
||||
engine.update_weights({"version_id": version_id})
|
||||
|
||||
assert (
|
||||
mx_client.stage_weight.call_args.kwargs["version"].version_id == version_id
|
||||
)
|
||||
mx_client.apply_weight.assert_called_with(staged)
|
||||
torch.accelerator.synchronize.assert_called()
|
||||
staged.release.assert_not_called()
|
||||
engine.finish_weight_update()
|
||||
staged.release.assert_called_once_with()
|
||||
|
||||
engine.shutdown()
|
||||
engine.shutdown()
|
||||
mx_client.close.assert_called_once_with()
|
||||
|
||||
|
||||
def test_vime_init_config_reaches_mx_client(mx_client):
|
||||
from modelexpress_rl.inference.client import ModelExpressGeneratorClient
|
||||
|
||||
engine = _make_modelexpress_engine()
|
||||
ModelExpressGeneratorClient.initialize.assert_not_called()
|
||||
engine.init_transfer_engine(
|
||||
engine.parse_init_info(
|
||||
{
|
||||
"model_name": "policy",
|
||||
"server_url": "mx:8001",
|
||||
"initial_serving_version_id": "serving-a",
|
||||
"object_storage_type": "S3",
|
||||
"initial_base_version_id": "base-a",
|
||||
"seed_checkpoint_path": "/models/launch",
|
||||
"refit_checkpoint_dir": "/mxdelta/receiver",
|
||||
"refit_checkpoint_max_size_gb": 200,
|
||||
"object_storage_endpoint_url": "http://minio:9000",
|
||||
"object_storage_region_name": "us-west-2",
|
||||
"registration_ttl_seconds": 90,
|
||||
"lease_ttl_seconds": 60,
|
||||
"max_transfer_attempts": 4,
|
||||
"max_replay_chain_length": 17,
|
||||
"rpc_timeout_seconds": 12.5,
|
||||
}
|
||||
)
|
||||
)
|
||||
config = ModelExpressGeneratorClient.initialize.call_args.args[0]
|
||||
assert config.engine_context.model is engine.model
|
||||
assert config.engine_context.vllm_config is engine.vllm_config
|
||||
assert config.model_name == "policy"
|
||||
assert config.server_url == "mx:8001"
|
||||
assert config.initial_serving_version_id == "serving-a"
|
||||
assert config.registration_ttl_seconds == 90
|
||||
assert config.lease_ttl_seconds == 60
|
||||
assert config.max_transfer_attempts == 4
|
||||
assert config.max_replay_chain_length == 17
|
||||
assert config.rpc_timeout_seconds == 12.5
|
||||
storage = config.object_storage
|
||||
assert storage.storage_type.value == "S3"
|
||||
assert storage.initial_base_version_id == "base-a"
|
||||
assert storage.seed_checkpoint_path == "/models/launch"
|
||||
assert storage.refit_checkpoint_dir == "/mxdelta/receiver"
|
||||
assert storage.refit_checkpoint_max_size_gb == 200
|
||||
assert storage.endpoint_url == "http://minio:9000"
|
||||
assert storage.region_name == "us-west-2"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"key",
|
||||
[
|
||||
"object_storage_type",
|
||||
"initial_base_version_id",
|
||||
"seed_checkpoint_path",
|
||||
"refit_checkpoint_dir",
|
||||
"object_storage_endpoint_url",
|
||||
"object_storage_region_name",
|
||||
],
|
||||
)
|
||||
def test_incomplete_storage_config_rejected_before_init(mx_client, key):
|
||||
from modelexpress_rl.inference.client import ModelExpressGeneratorClient
|
||||
|
||||
engine = _make_modelexpress_engine()
|
||||
with pytest.raises(ValueError, match="object storage requires"):
|
||||
engine.init_transfer_engine(engine.parse_init_info({key: "value"}))
|
||||
ModelExpressGeneratorClient.initialize.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("version_id", ["", " ", None, 1])
|
||||
def test_invalid_version_rejected_before_staging(mx_client, version_id):
|
||||
engine = _make_modelexpress_engine()
|
||||
engine.init_transfer_engine(engine.parse_init_info({}))
|
||||
engine.start_weight_update()
|
||||
with pytest.raises(ValueError, match="version_id is required"):
|
||||
engine.update_weights({"version_id": version_id})
|
||||
mx_client.stage_weight.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failure", ["stage", "apply", "release"])
|
||||
def test_update_failure_preserves_error_and_releases_handle(mx_client, failure):
|
||||
engine = _make_modelexpress_engine()
|
||||
engine.init_transfer_engine(engine.parse_init_info({}))
|
||||
staged = SimpleNamespace(version_id="version-a", metrics={}, release=MagicMock())
|
||||
mx_client.stage_weight.return_value = staged
|
||||
if failure == "stage":
|
||||
mx_client.stage_weight.side_effect = RuntimeError("stage failed")
|
||||
else:
|
||||
mx_client.apply_weight.side_effect = RuntimeError("apply failed")
|
||||
if failure == "release":
|
||||
staged.release.side_effect = RuntimeError("release failed")
|
||||
engine.start_weight_update()
|
||||
with pytest.raises(RuntimeError, match="stage failed|apply failed"):
|
||||
engine.update_weights({"version_id": "version-a"})
|
||||
assert staged.release.call_count == (0 if failure == "stage" else 1)
|
||||
torch.accelerator.synchronize.assert_not_called()
|
||||
engine.shutdown()
|
||||
mx_client.close.assert_called_once_with()
|
||||
|
||||
|
||||
def test_plugin_does_not_replace_native_backend(mx_client):
|
||||
from modelexpress.engines.vllm.registration import (
|
||||
register_plugin_weight_transfer_engine,
|
||||
)
|
||||
from modelexpress_rl.inference.engines.vllm.weight_transfer_engine import (
|
||||
ModelExpressWeightTransferEngine,
|
||||
)
|
||||
|
||||
native_loader = WeightTransferEngineFactory._registry["modelexpress"]
|
||||
register_plugin_weight_transfer_engine()
|
||||
assert WeightTransferEngineFactory._registry["modelexpress"] is native_loader
|
||||
assert type(_make_modelexpress_engine()) is ModelExpressWeightTransferEngine
|
||||
|
||||
@@ -27,6 +27,7 @@ from pydantic import BaseModel, Field, ValidationError
|
||||
|
||||
from vllm.entrypoints.anthropic.api_router import attach_router
|
||||
from vllm.entrypoints.anthropic.protocol import (
|
||||
AnthropicCountTokensRequest,
|
||||
AnthropicMessagesRequest,
|
||||
)
|
||||
from vllm.entrypoints.anthropic.serving import (
|
||||
@@ -37,6 +38,8 @@ from vllm.entrypoints.generate.base.protocol import (
|
||||
DeltaFunctionCall,
|
||||
DeltaMessage,
|
||||
DeltaToolCall,
|
||||
FunctionCall,
|
||||
ToolCall,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionResponse,
|
||||
@@ -353,9 +356,11 @@ class TestToolResultContent:
|
||||
if m["role"] == "user" and isinstance(m.get("content"), list)
|
||||
]
|
||||
assert len(follow_up) == 1
|
||||
assert follow_up[0]["content"][0]["image_url"]["url"] == (
|
||||
"data:image/jpeg;base64,QUFB"
|
||||
)
|
||||
content = follow_up[0]["content"]
|
||||
assert isinstance(content, list)
|
||||
image = content[0]
|
||||
assert image["type"] == "image_url"
|
||||
assert image["image_url"]["url"] == "data:image/jpeg;base64,QUFB"
|
||||
|
||||
def test_tool_result_with_multiple_images(self):
|
||||
request = self._make_tool_result_request(
|
||||
@@ -385,7 +390,12 @@ class TestToolResultContent:
|
||||
if m["role"] == "user" and isinstance(m.get("content"), list)
|
||||
]
|
||||
assert len(follow_up) == 1
|
||||
urls = [p["image_url"]["url"] for p in follow_up[0]["content"]]
|
||||
content = follow_up[0]["content"]
|
||||
assert isinstance(content, list)
|
||||
urls = []
|
||||
for part in content:
|
||||
assert part["type"] == "image_url"
|
||||
urls.append(part["image_url"]["url"])
|
||||
assert urls == [
|
||||
"data:image/png;base64,IMG1",
|
||||
"https://example.com/img2.jpg",
|
||||
@@ -582,7 +592,8 @@ class TestThinkingBlockConversion:
|
||||
asst = asst_msgs[0]
|
||||
|
||||
assert asst.get("reasoning") == "I need to call the calculator."
|
||||
tool_calls = list(asst.get("tool_calls", []))
|
||||
tool_calls = asst.get("tool_calls")
|
||||
assert tool_calls is not None
|
||||
assert len(tool_calls) == 1
|
||||
assert tool_calls[0]["function"]["name"] == "calculator"
|
||||
# No text content alongside reasoning + tool_use.
|
||||
@@ -999,11 +1010,6 @@ class TestInlineSystemMessageInMessagesArray:
|
||||
|
||||
def _make_stream_converter():
|
||||
obj = MagicMock(spec=AnthropicServingMessages)
|
||||
obj.stop_reason_map = {
|
||||
"stop": "end_turn",
|
||||
"length": "max_tokens",
|
||||
"tool_calls": "tool_use",
|
||||
}
|
||||
obj.message_stream_converter = (
|
||||
AnthropicServingMessages.message_stream_converter.__get__(obj)
|
||||
)
|
||||
@@ -1445,7 +1451,7 @@ class TestDetectMergeInlineSystem:
|
||||
AnthropicServingMessages._should_merge_inline_system(online_renderer)
|
||||
is expected
|
||||
)
|
||||
assert mock_logger.warning.called is expected
|
||||
assert mock_logger.warning_once.called is expected
|
||||
|
||||
|
||||
# ======================================================================
|
||||
@@ -1671,6 +1677,78 @@ class TestStopSequenceReason:
|
||||
assert msg_deltas[0]["delta"]["stop_sequence"] is None
|
||||
|
||||
|
||||
class TestToolUseStopReason:
|
||||
"""A response carrying a tool_use block reports ``stop_reason="tool_use"``
|
||||
even when the chat layer says ``finish_reason="stop"``, as it does for a
|
||||
forced (named) tool. A call cut short by max_tokens or a stop string does not.
|
||||
"""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("finish_reason", "stop_reason", "expected"),
|
||||
[
|
||||
("stop", None, "tool_use"),
|
||||
("tool_calls", "</x>", "tool_use"),
|
||||
],
|
||||
)
|
||||
def test_non_streaming(self, finish_reason, stop_reason, expected):
|
||||
tool_call = ToolCall(
|
||||
function=FunctionCall(name="get_weather", arguments='{"city": "Paris"}')
|
||||
)
|
||||
response = ChatCompletionResponse(
|
||||
id="chatcmpl-test",
|
||||
model="test-model",
|
||||
choices=[
|
||||
ChatCompletionResponseChoice(
|
||||
index=0,
|
||||
message=ChatMessage(role="assistant", tool_calls=[tool_call]),
|
||||
finish_reason=finish_reason,
|
||||
stop_reason=stop_reason,
|
||||
)
|
||||
],
|
||||
usage=UsageInfo(prompt_tokens=5, total_tokens=8, completion_tokens=3),
|
||||
)
|
||||
|
||||
result = _make_full_converter().messages_full_converter(response)
|
||||
|
||||
assert result.content[0].type == "tool_use"
|
||||
assert result.stop_reason == expected
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("args", "finish_reason", "stop_reason", "expected"),
|
||||
[
|
||||
('{"city": "Paris"}', "stop", None, "tool_use"),
|
||||
('{"city": "Pa', "length", None, "max_tokens"),
|
||||
('{"city": "Paris"}', "tool_calls", "</x>", "tool_use"),
|
||||
('{"city": "Pa', "tool_calls", "</x>", "stop_sequence"),
|
||||
],
|
||||
)
|
||||
async def test_streaming(self, args, finish_reason, stop_reason, expected):
|
||||
async def sse_input():
|
||||
yield _make_stream_chunk(delta=DeltaMessage(role="assistant"))
|
||||
yield _make_stream_chunk(
|
||||
delta=DeltaMessage(
|
||||
tool_calls=[_tc(id="call_1", name="get_weather", args=args)]
|
||||
)
|
||||
)
|
||||
yield _make_stream_chunk(
|
||||
finish_reason=finish_reason, stop_reason=stop_reason
|
||||
)
|
||||
yield _make_stream_chunk(
|
||||
choices=[],
|
||||
usage=UsageInfo(prompt_tokens=5, total_tokens=8, completion_tokens=3),
|
||||
)
|
||||
yield "data: [DONE]"
|
||||
|
||||
converter = _make_stream_converter()
|
||||
events = _parse_sse_events(
|
||||
[e async for e in converter.message_stream_converter(sse_input())]
|
||||
)
|
||||
|
||||
msg_deltas = [data for ev_type, data in events if ev_type == "message_delta"]
|
||||
assert msg_deltas[0]["delta"]["stop_reason"] == expected
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# Client-caused errors are 4xx, not 500 (Issue #52088)
|
||||
# ======================================================================
|
||||
@@ -1955,3 +2033,87 @@ class TestProbeDisabledThinkingEffort:
|
||||
@pytest.mark.asyncio
|
||||
async def test_renderer_rejects_none(self):
|
||||
assert await self._probe(self._reject_none) == "low"
|
||||
|
||||
|
||||
class TestMidConversationToolChanges:
|
||||
"""``tool_addition``/``tool_removal`` blocks decide which tools the chat
|
||||
template is given, since it has no other way to see them."""
|
||||
|
||||
@staticmethod
|
||||
def _convert_tools(*changes: dict) -> dict[str, bool | None]:
|
||||
request = _make_request(
|
||||
[
|
||||
{"role": "user", "content": "Hello"},
|
||||
{
|
||||
"role": "system",
|
||||
"content": [{"type": "text", "text": "Tools changed."}, *changes],
|
||||
},
|
||||
],
|
||||
tools=[
|
||||
{"name": "bash", "input_schema": {}},
|
||||
{"name": "search", "input_schema": {}, "defer_loading": True},
|
||||
{"name": "fetch", "input_schema": {}, "defer_loading": True},
|
||||
],
|
||||
)
|
||||
result = _convert(request)
|
||||
assert result.messages[-1] == {"role": "system", "content": "Tools changed."}
|
||||
assert result.tools is not None
|
||||
return {t.function.name: t.function.defer_loading for t in result.tools}
|
||||
|
||||
@staticmethod
|
||||
def _change(block_type: str, name: str) -> dict:
|
||||
return {"type": block_type, "tool": {"type": "tool_reference", "name": name}}
|
||||
|
||||
def test_addition_loads_deferred_tool(self):
|
||||
tools = self._convert_tools(self._change("tool_addition", "search"))
|
||||
assert tools == {"bash": None, "search": None, "fetch": True}
|
||||
|
||||
def test_removal_withdraws_tool_until_added_again(self):
|
||||
removal = self._change("tool_removal", "bash")
|
||||
assert "bash" not in self._convert_tools(removal)
|
||||
readded = self._convert_tools(removal, self._change("tool_addition", "bash"))
|
||||
assert list(readded) == ["bash", "search", "fetch"]
|
||||
|
||||
def test_addition_defines_tool_by_value(self):
|
||||
definition = {"name": "db_query", "input_schema": {}}
|
||||
tools = self._convert_tools(
|
||||
{
|
||||
"type": "tool_addition",
|
||||
"tool": {"type": "tool_definition", "definition": definition},
|
||||
}
|
||||
)
|
||||
assert list(tools) == ["bash", "search", "fetch", "db_query"]
|
||||
|
||||
def test_unknown_tool_reference_is_rejected(self):
|
||||
with pytest.raises(ValueError, match="not declared in tools"):
|
||||
self._convert_tools(self._change("tool_addition", "missing"))
|
||||
|
||||
def test_removal_by_value_is_rejected(self):
|
||||
definition = {"name": "bash", "input_schema": {}}
|
||||
with pytest.raises(ValidationError, match="only accepts a tool_reference"):
|
||||
self._convert_tools(
|
||||
{
|
||||
"type": "tool_removal",
|
||||
"tool": {"type": "tool_definition", "definition": definition},
|
||||
}
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"request_cls", [AnthropicMessagesRequest, AnthropicCountTokensRequest]
|
||||
)
|
||||
@pytest.mark.parametrize("in_top_level_system", [False, True])
|
||||
def test_tool_change_outside_system_message_is_rejected(
|
||||
self, request_cls, in_top_level_system
|
||||
):
|
||||
change = self._change("tool_addition", "search")
|
||||
if in_top_level_system:
|
||||
fields = {
|
||||
"messages": [{"role": "user", "content": "Hi"}],
|
||||
"system": [change],
|
||||
}
|
||||
else:
|
||||
fields = {"messages": [{"role": "user", "content": [change]}]}
|
||||
with pytest.raises(
|
||||
ValidationError, match='only allowed in messages with role "system"'
|
||||
):
|
||||
request_cls(model="test-model", max_tokens=128, **fields)
|
||||
|
||||
@@ -224,7 +224,7 @@ async def test_anthropic_streaming_cache_usage(client: anthropic.AsyncAnthropic)
|
||||
request = dict(
|
||||
model="claude-3-7-sonnet-latest",
|
||||
max_tokens=1,
|
||||
temperature=0.0,
|
||||
extra_body={"temperature": 0.0},
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
|
||||
@@ -29,7 +29,7 @@ from vllm.entrypoints.cohere.cohere_chat_message import (
|
||||
class TestCitationSource:
|
||||
def test_invalid_type_rejected(self):
|
||||
with pytest.raises(ValidationError):
|
||||
CitationSource(type="other") # type: ignore[arg-type]
|
||||
CitationSource(type="other")
|
||||
|
||||
def test_none_fields_excluded_from_dump(self):
|
||||
s = CitationSource(type="document", id="d1")
|
||||
@@ -65,7 +65,7 @@ class TestCitationSource:
|
||||
class TestCitation:
|
||||
def test_invalid_type_rejected(self):
|
||||
with pytest.raises(ValidationError):
|
||||
Citation(type="OTHER") # type: ignore[arg-type]
|
||||
Citation(type="OTHER")
|
||||
|
||||
def test_dump_excludes_none_fields(self):
|
||||
c = Citation(start=0, end=5, text="hello")
|
||||
|
||||
@@ -136,7 +136,7 @@ class TestCohereChatV2Response:
|
||||
with pytest.raises(ValidationError):
|
||||
CohereChatV2Response(
|
||||
id="r1",
|
||||
finish_reason="NOT_A_REASON", # type: ignore[arg-type]
|
||||
finish_reason="NOT_A_REASON",
|
||||
message=msg,
|
||||
)
|
||||
|
||||
@@ -246,5 +246,5 @@ class TestStreamingEventTypeField:
|
||||
MessageStartEvent(
|
||||
id="a",
|
||||
delta={"message": {"role": "assistant"}},
|
||||
type="other", # type: ignore[arg-type]
|
||||
type="other",
|
||||
)
|
||||
|
||||
@@ -14,16 +14,6 @@ def sample_prompts():
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_token_ids():
|
||||
return [
|
||||
[0],
|
||||
[0, 1],
|
||||
[0, 2, 1],
|
||||
[0, 3, 1, 2],
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_regex():
|
||||
return (
|
||||
@@ -61,104 +51,6 @@ def sample_json_schema():
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_complex_json_schema():
|
||||
return {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"score": {
|
||||
"type": "integer",
|
||||
"minimum": 0,
|
||||
"maximum": 100, # Numeric range
|
||||
},
|
||||
"grade": {
|
||||
"type": "string",
|
||||
"pattern": "^[A-D]$", # Regex pattern
|
||||
},
|
||||
"email": {
|
||||
"type": "string",
|
||||
"pattern": "^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}$",
|
||||
},
|
||||
"tags": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string",
|
||||
# Combining length and pattern restrictions
|
||||
"pattern": "^[a-z]{1,10}$",
|
||||
},
|
||||
},
|
||||
},
|
||||
"required": ["score", "grade", "email", "tags"],
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_definition_json_schema():
|
||||
return {
|
||||
"$defs": {
|
||||
"Step": {
|
||||
"properties": {
|
||||
"explanation": {"title": "Explanation", "type": "string"},
|
||||
"output": {"title": "Output", "type": "string"},
|
||||
},
|
||||
"required": ["explanation", "output"],
|
||||
"title": "Step",
|
||||
"type": "object",
|
||||
}
|
||||
},
|
||||
"properties": {
|
||||
"steps": {
|
||||
"items": {"$ref": "#/$defs/Step"},
|
||||
"title": "Steps",
|
||||
"type": "array",
|
||||
},
|
||||
"final_answer": {"title": "Final Answer", "type": "string"},
|
||||
},
|
||||
"required": ["steps", "final_answer"],
|
||||
"title": "MathReasoning",
|
||||
"type": "object",
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_enum_json_schema():
|
||||
return {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"status": {
|
||||
"type": "string",
|
||||
"enum": ["active", "inactive", "pending"], # Literal values using enum
|
||||
},
|
||||
"priority": {
|
||||
"type": "string",
|
||||
"enum": ["low", "medium", "high", "critical"],
|
||||
},
|
||||
"category": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"type": {
|
||||
"type": "string",
|
||||
"enum": ["bug", "feature", "improvement"],
|
||||
},
|
||||
"severity": {
|
||||
"type": "integer",
|
||||
"enum": [1, 2, 3, 4, 5], # Enum can also contain numbers
|
||||
},
|
||||
},
|
||||
"required": ["type", "severity"],
|
||||
},
|
||||
"flags": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string",
|
||||
"enum": ["urgent", "blocked", "needs_review", "approved"],
|
||||
},
|
||||
},
|
||||
},
|
||||
"required": ["status", "priority", "category", "flags"],
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_structured_outputs_choices():
|
||||
return [
|
||||
@@ -175,18 +67,6 @@ def sample_structured_outputs_choices():
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_sql_statements():
|
||||
return """
|
||||
start: select_statement
|
||||
select_statement: "SELECT" column "from" table "where" condition
|
||||
column: "col_1" | "col_2"
|
||||
table: "table_1" | "table_2"
|
||||
condition: column "=" number
|
||||
number: "1" | "2"
|
||||
"""
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def qwen3_lora_files():
|
||||
"""Download Qwen3 LoRA files once per test session."""
|
||||
|
||||
@@ -307,7 +307,8 @@ def test_external_process_monitoring(api_server_args):
|
||||
def run_with_exception_capture():
|
||||
try:
|
||||
wait_for_completion_or_failure(
|
||||
api_server_manager=manager, coordinator=mock_coordinator
|
||||
api_server_manager=manager,
|
||||
coordinator=mock_coordinator,
|
||||
)
|
||||
except Exception as e:
|
||||
result["exception"] = e
|
||||
@@ -489,3 +490,102 @@ def test_rust_frontend_launch_log_redacts_credentials(monkeypatch, caplog):
|
||||
assert hf_token not in message
|
||||
assert api_key not in message
|
||||
assert '"hf_token": "***"' in message
|
||||
|
||||
|
||||
def test_rust_frontend_uses_config_model_as_model_tag(monkeypatch, caplog, tmp_path):
|
||||
"""Config-only model selection must be forwarded to the Rust frontend."""
|
||||
import subprocess as subprocess_mod
|
||||
|
||||
from vllm.entrypoints.launchers.cli_args import make_arg_parser
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
from vllm.v1.utils import RustFrontendProcessManager
|
||||
|
||||
config_path = tmp_path / "config.yml"
|
||||
config_path.write_text("model: org/model\n")
|
||||
args = make_arg_parser(FlexibleArgumentParser()).parse_args(
|
||||
["--config", str(config_path)]
|
||||
)
|
||||
assert args.model == "org/model"
|
||||
assert args.model_tag is None
|
||||
|
||||
class _FakeProc:
|
||||
pid = 4321
|
||||
returncode = 0
|
||||
|
||||
def poll(self):
|
||||
return 0
|
||||
|
||||
def wait(self, timeout=None):
|
||||
return 0
|
||||
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
try:
|
||||
monkeypatch.setattr(subprocess_mod, "Popen", lambda *a, **kw: _FakeProc())
|
||||
with caplog.at_level("INFO", logger="vllm.v1.utils"):
|
||||
RustFrontendProcessManager(
|
||||
binary_path="/nonexistent/vllm-rs",
|
||||
sock=sock,
|
||||
args=args,
|
||||
input_address="ipc:///tmp/in",
|
||||
output_address="ipc:///tmp/out",
|
||||
engine_start_index=0,
|
||||
engine_count=1,
|
||||
data_parallel_size=1,
|
||||
)
|
||||
finally:
|
||||
sock.close()
|
||||
|
||||
assert '"model_tag": "org/model"' in caplog.text
|
||||
|
||||
|
||||
def test_rust_frontend_inherits_grpc_listener(monkeypatch):
|
||||
"""The Python-bound gRPC socket is passed to Rust like the HTTP one."""
|
||||
import subprocess as subprocess_mod
|
||||
|
||||
from vllm.entrypoints.launchers.cli_args import make_arg_parser
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
from vllm.v1.utils import RustFrontendProcessManager
|
||||
|
||||
args = make_arg_parser(FlexibleArgumentParser()).parse_args(
|
||||
["--model", "org/model", "--grpc-port", "50051"]
|
||||
)
|
||||
|
||||
class _FakeProc:
|
||||
pid = 4321
|
||||
returncode = 0
|
||||
|
||||
def poll(self):
|
||||
return 0
|
||||
|
||||
def wait(self, timeout=None):
|
||||
return 0
|
||||
|
||||
popen_calls = []
|
||||
|
||||
def fake_popen(cmd, **kwargs):
|
||||
popen_calls.append((cmd, kwargs))
|
||||
return _FakeProc()
|
||||
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
grpc_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
try:
|
||||
monkeypatch.setattr(subprocess_mod, "Popen", fake_popen)
|
||||
RustFrontendProcessManager(
|
||||
binary_path="/nonexistent/vllm-rs",
|
||||
sock=sock,
|
||||
args=args,
|
||||
input_address="ipc:///tmp/in",
|
||||
output_address="ipc:///tmp/out",
|
||||
engine_start_index=0,
|
||||
engine_count=1,
|
||||
data_parallel_size=1,
|
||||
grpc_sock=grpc_sock,
|
||||
)
|
||||
[(cmd, kwargs)] = popen_calls
|
||||
grpc_fd = grpc_sock.fileno()
|
||||
flag_index = cmd.index("--grpc-listen-fd")
|
||||
assert cmd[flag_index + 1] == str(grpc_fd)
|
||||
assert list(kwargs["pass_fds"]) == [sock.fileno(), grpc_fd]
|
||||
finally:
|
||||
sock.close()
|
||||
grpc_sock.close()
|
||||
|
||||
@@ -28,7 +28,7 @@ assert CHATML_JINJA_PATH.exists()
|
||||
|
||||
def _build_vllm_parsers():
|
||||
vllm_parser = FlexibleArgumentParser()
|
||||
subparsers = vllm_parser.add_subparsers()
|
||||
subparsers = vllm_parser.add_subparsers(dest="subparser")
|
||||
serve_parser = subparsers.add_parser("serve")
|
||||
make_arg_parser(serve_parser)
|
||||
return {"vllm": vllm_parser, "vllm serve": serve_parser}
|
||||
@@ -76,6 +76,8 @@ def test_logging_config_cli_args(serve_parser):
|
||||
[
|
||||
"--logging-config",
|
||||
'{"log_level":"WARNING","pylogging_config_file":"/tmp/json.json"}',
|
||||
"--logging-config.formatter",
|
||||
"json",
|
||||
"--log-level",
|
||||
"DEBUG",
|
||||
"--log-config-file",
|
||||
@@ -85,6 +87,7 @@ def test_logging_config_cli_args(serve_parser):
|
||||
|
||||
config = AsyncEngineArgs.from_cli_args(args).create_logging_config()
|
||||
assert config.log_level == "DEBUG"
|
||||
assert config.formatter == "json"
|
||||
assert config.pylogging_config_file == "/tmp/flat.json"
|
||||
|
||||
|
||||
@@ -170,6 +173,75 @@ def test_multiple_valid_inputs(serve_parser):
|
||||
|
||||
|
||||
### Tests for serve argument validation that run prior to loading
|
||||
@pytest.mark.parametrize(
|
||||
"port, extra_args",
|
||||
[
|
||||
(50051, []),
|
||||
(0, ["--port", "0"]),
|
||||
(50051, ["--port", "50051", "--uds", "/tmp/vllm.sock"]),
|
||||
],
|
||||
)
|
||||
def test_rust_grpc_port_accepts_frontend_launch(
|
||||
vllm_parser, monkeypatch, port, extra_args
|
||||
):
|
||||
monkeypatch.setenv("VLLM_USE_RUST_FRONTEND", "1")
|
||||
args = vllm_parser.parse_args(
|
||||
[
|
||||
"serve",
|
||||
"--grpc-port",
|
||||
str(port),
|
||||
"--data-parallel-size",
|
||||
"8",
|
||||
"--data-parallel-size-local",
|
||||
"4",
|
||||
"--data-parallel-start-rank",
|
||||
"4",
|
||||
"--data-parallel-hybrid-lb",
|
||||
*extra_args,
|
||||
]
|
||||
)
|
||||
validate_parsed_serve_args(args)
|
||||
assert args.grpc_port == port
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"rust_enabled, extra_args, port",
|
||||
[
|
||||
(False, [], 50051),
|
||||
(False, ["--grpc"], 50051),
|
||||
(True, ["--grpc"], 50051),
|
||||
(True, ["--headless"], 50051),
|
||||
(True, ["--api-server-count", "0"], 50051),
|
||||
(True, ["--api-server-count", "-1"], 50051),
|
||||
(True, ["--data-parallel-multi-port-external-lb"], 50051),
|
||||
(True, ["--port", "50051"], 50051),
|
||||
(True, [], -1),
|
||||
(True, [], 65536),
|
||||
],
|
||||
)
|
||||
def test_rust_grpc_port_rejects_incompatible_launch(
|
||||
vllm_parser, monkeypatch, rust_enabled, extra_args, port
|
||||
):
|
||||
monkeypatch.setenv("VLLM_USE_RUST_FRONTEND", "1" if rust_enabled else "0")
|
||||
args = vllm_parser.parse_args(["serve", "--grpc-port", str(port), *extra_args])
|
||||
error = (
|
||||
"--grpc and --grpc-port are mutually exclusive"
|
||||
if "--grpc" in extra_args
|
||||
else "--grpc-port"
|
||||
)
|
||||
with pytest.raises(ValueError, match=error):
|
||||
validate_parsed_serve_args(args)
|
||||
|
||||
|
||||
def test_rust_grpc_port_requires_serve_subcommand(monkeypatch):
|
||||
"""Direct Python render/API entrypoints must reject a Rust-only listener."""
|
||||
monkeypatch.setenv("VLLM_USE_RUST_FRONTEND", "1")
|
||||
parser = make_arg_parser(FlexibleArgumentParser())
|
||||
args = parser.parse_args(["--grpc-port", "50051"])
|
||||
with pytest.raises(ValueError, match="--grpc-port requires"):
|
||||
validate_parsed_serve_args(args)
|
||||
|
||||
|
||||
def test_enable_auto_choice_passes_without_tool_call_parser(serve_parser):
|
||||
"""Ensure validation fails if tool choice is enabled with no call parser."""
|
||||
# If we enable-auto-tool-choice, explode with no tool-call-parser
|
||||
@@ -262,13 +334,21 @@ def launch_render_parser():
|
||||
return _build_launch_render_parser()
|
||||
|
||||
|
||||
def test_launch_render_validates_serve_args(launch_render_parser):
|
||||
@pytest.mark.parametrize(
|
||||
"cli_args, error",
|
||||
[
|
||||
(["--enable-auto-tool-choice"], TypeError),
|
||||
(["--grpc-port", "50051"], ValueError),
|
||||
],
|
||||
)
|
||||
def test_launch_render_validates_serve_args(
|
||||
launch_render_parser, monkeypatch, cli_args, error
|
||||
):
|
||||
"""`vllm launch render` reuses the serve parser, so it gets the serve checks"""
|
||||
args = launch_render_parser.parse_args(
|
||||
args=["launch", "render", "--enable-auto-tool-choice"]
|
||||
)
|
||||
monkeypatch.setenv("VLLM_USE_RUST_FRONTEND", "1")
|
||||
args = launch_render_parser.parse_args(args=["launch", "render", *cli_args])
|
||||
assert args.subparser == "launch"
|
||||
with pytest.raises(TypeError):
|
||||
with pytest.raises(error):
|
||||
validate_parsed_serve_args(args)
|
||||
|
||||
|
||||
|
||||
@@ -47,7 +47,6 @@ _SUPERVISOR_PORT = 19256
|
||||
_CHILD_PORT_BASE = 18000
|
||||
_N_CHILDREN = 2
|
||||
_PROBE_INTERVAL = 1.0
|
||||
_POLL_INTERVAL = 1.0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -14,6 +14,7 @@ import httpx
|
||||
import openai
|
||||
import psutil
|
||||
import pytest
|
||||
from prometheus_client.parser import text_string_to_metric_families
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
@@ -127,6 +128,40 @@ async def _concurrent_request_loop(
|
||||
t.cancel()
|
||||
|
||||
|
||||
def _num_running_requests(metrics_text: str) -> float:
|
||||
"""Sum ``vllm:num_requests_running`` across the server's label sets."""
|
||||
total = 0.0
|
||||
for family in text_string_to_metric_families(metrics_text):
|
||||
if family.name == "vllm:num_requests_running":
|
||||
for sample in family.samples:
|
||||
if sample.name == "vllm:num_requests_running":
|
||||
total += sample.value
|
||||
return total
|
||||
|
||||
|
||||
async def _wait_for_running_request(
|
||||
metrics_url: str,
|
||||
timeout: float = _INFLIGHT_REQUEST_START_TIMEOUT,
|
||||
) -> float:
|
||||
"""Wait until the server reports a running request, returning the delay.
|
||||
|
||||
``ShutdownState.inflight_requests`` is incremented before the HTTP call is
|
||||
awaited, so it only shows that a client coroutine started. Draining can
|
||||
only complete work the engine has already admitted, so synchronize on the
|
||||
server's own scheduler gauge instead.
|
||||
"""
|
||||
start = time.monotonic()
|
||||
deadline = start + timeout
|
||||
async with httpx.AsyncClient() as metrics_client:
|
||||
while time.monotonic() < deadline:
|
||||
response = await metrics_client.get(metrics_url)
|
||||
response.raise_for_status()
|
||||
if _num_running_requests(response.text) > 0:
|
||||
return time.monotonic() - start
|
||||
await asyncio.sleep(_INFLIGHT_REQUEST_POLL_INTERVAL)
|
||||
raise AssertionError(f"No request reached the running state within {timeout}s")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_shutdown_on_engine_failure(tmp_path: Path):
|
||||
"""Verify that API returns connection error when server process is killed.
|
||||
@@ -240,11 +275,15 @@ async def test_wait_timeout_completes_requests():
|
||||
_concurrent_request_loop(client, state, sigterm_sent, concurrency=10)
|
||||
)
|
||||
|
||||
await asyncio.sleep(0.5)
|
||||
proc.send_signal(signal.SIGTERM)
|
||||
sigterm_sent.set()
|
||||
|
||||
try:
|
||||
# Drain can only complete requests the engine already admitted, so
|
||||
# wait for the server to report one instead of guessing with a sleep.
|
||||
admission_delay = await _wait_for_running_request(
|
||||
remote_server.url_for("metrics")
|
||||
)
|
||||
proc.send_signal(signal.SIGTERM)
|
||||
sigterm_sent.set()
|
||||
|
||||
await asyncio.wait_for(request_task, timeout=_SHUTDOWN_DETECTION_TIMEOUT)
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
@@ -257,6 +296,7 @@ async def test_wait_timeout_completes_requests():
|
||||
# wait timeout should complete in-flight requests
|
||||
assert state.requests_after_sigterm > 0, (
|
||||
f"Wait timeout should complete in-flight requests. "
|
||||
f"admission_delay: {admission_delay:.3f}s, "
|
||||
f"503: {state.got_503}, 500: {state.got_500}, "
|
||||
f"conn_errors: {state.connection_errors}, errors: {state.errors}"
|
||||
)
|
||||
|
||||
@@ -7,6 +7,9 @@ from unittest.mock import Mock
|
||||
import pytest
|
||||
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
|
||||
from vllm.inputs import TextPrompt
|
||||
from vllm.outputs import RequestOutput
|
||||
|
||||
|
||||
def _make_mock_llm() -> LLM:
|
||||
@@ -17,12 +20,13 @@ def _make_mock_llm() -> LLM:
|
||||
return llm
|
||||
|
||||
|
||||
def test_generate_forwards_mm_processor_kwargs() -> None:
|
||||
def test_generate_forwards_mm_processor_kwargs(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
llm = _make_mock_llm()
|
||||
mm_processor_kwargs = {"num_crops": 4}
|
||||
sampling_params = SamplingParams(max_tokens=1)
|
||||
|
||||
llm._run_completion = Mock(return_value=["ok"])
|
||||
_run_completion = Mock(return_value=["ok"])
|
||||
monkeypatch.setattr(llm, "_run_completion", _run_completion)
|
||||
|
||||
outputs = llm.generate(
|
||||
"prompt",
|
||||
@@ -31,17 +35,18 @@ def test_generate_forwards_mm_processor_kwargs() -> None:
|
||||
)
|
||||
|
||||
assert outputs == ["ok"]
|
||||
assert llm._run_completion.call_args.kwargs["mm_processor_kwargs"] == (
|
||||
assert _run_completion.call_args.kwargs["mm_processor_kwargs"] == (
|
||||
mm_processor_kwargs
|
||||
)
|
||||
|
||||
|
||||
def test_enqueue_forwards_mm_processor_kwargs() -> None:
|
||||
def test_enqueue_forwards_mm_processor_kwargs(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
llm = _make_mock_llm()
|
||||
mm_processor_kwargs = {"do_resize": False}
|
||||
sampling_params = SamplingParams(max_tokens=1)
|
||||
|
||||
llm._add_completion_requests = Mock(return_value=["req-0"])
|
||||
_add_completion_requests = Mock(return_value=["req-0"])
|
||||
monkeypatch.setattr(llm, "_add_completion_requests", _add_completion_requests)
|
||||
|
||||
request_ids = llm.enqueue(
|
||||
"prompt",
|
||||
@@ -51,18 +56,19 @@ def test_enqueue_forwards_mm_processor_kwargs() -> None:
|
||||
)
|
||||
|
||||
assert request_ids == ["req-0"]
|
||||
assert llm._add_completion_requests.call_args.kwargs["mm_processor_kwargs"] == (
|
||||
assert _add_completion_requests.call_args.kwargs["mm_processor_kwargs"] == (
|
||||
mm_processor_kwargs
|
||||
)
|
||||
|
||||
|
||||
def test_chat_forwards_mm_processor_kwargs() -> None:
|
||||
def test_chat_forwards_mm_processor_kwargs(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
llm = _make_mock_llm()
|
||||
mm_processor_kwargs = {"do_pan_and_scan": True}
|
||||
sampling_params = SamplingParams(max_tokens=1)
|
||||
messages = [{"role": "user", "content": "hello"}]
|
||||
messages: list[ChatCompletionMessageParam] = [{"role": "user", "content": "hello"}]
|
||||
|
||||
llm._run_chat = Mock(return_value=["ok"])
|
||||
_run_chat = Mock(return_value=["ok"])
|
||||
monkeypatch.setattr(llm, "_run_chat", _run_chat)
|
||||
|
||||
outputs = llm.chat(
|
||||
messages,
|
||||
@@ -71,18 +77,19 @@ def test_chat_forwards_mm_processor_kwargs() -> None:
|
||||
)
|
||||
|
||||
assert outputs == ["ok"]
|
||||
assert llm._run_chat.call_args.kwargs["mm_processor_kwargs"] == (
|
||||
mm_processor_kwargs
|
||||
)
|
||||
assert _run_chat.call_args.kwargs["mm_processor_kwargs"] == mm_processor_kwargs
|
||||
|
||||
|
||||
def test_enqueue_chat_forwards_mm_processor_kwargs() -> None:
|
||||
def test_enqueue_chat_forwards_mm_processor_kwargs(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
llm = _make_mock_llm()
|
||||
mm_processor_kwargs = {"do_pan_and_scan": True}
|
||||
sampling_params = SamplingParams(max_tokens=1)
|
||||
messages = [{"role": "user", "content": "hello"}]
|
||||
messages: list[ChatCompletionMessageParam] = [{"role": "user", "content": "hello"}]
|
||||
|
||||
llm._add_chat_requests = Mock(return_value=["req-0"])
|
||||
_add_chat_requests = Mock(return_value=["req-0"])
|
||||
monkeypatch.setattr(llm, "_add_chat_requests", _add_chat_requests)
|
||||
|
||||
request_ids = llm.enqueue_chat(
|
||||
messages,
|
||||
@@ -92,67 +99,79 @@ def test_enqueue_chat_forwards_mm_processor_kwargs() -> None:
|
||||
)
|
||||
|
||||
assert request_ids == ["req-0"]
|
||||
assert llm._add_chat_requests.call_args.kwargs["mm_processor_kwargs"] == (
|
||||
assert _add_chat_requests.call_args.kwargs["mm_processor_kwargs"] == (
|
||||
mm_processor_kwargs
|
||||
)
|
||||
|
||||
|
||||
def test_run_chat_forwards_mm_processor_kwargs() -> None:
|
||||
def test_run_chat_forwards_mm_processor_kwargs(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
llm = _make_mock_llm()
|
||||
mm_processor_kwargs = {"num_crops": 8}
|
||||
sampling_params = SamplingParams(max_tokens=1)
|
||||
messages = [{"role": "user", "content": "hello"}]
|
||||
messages: list[ChatCompletionMessageParam] = [{"role": "user", "content": "hello"}]
|
||||
sentinel_output = ["done"]
|
||||
|
||||
llm._add_chat_requests = Mock()
|
||||
llm._run_engine = Mock(return_value=sentinel_output)
|
||||
_add_chat_requests = Mock()
|
||||
monkeypatch.setattr(llm, "_add_chat_requests", _add_chat_requests)
|
||||
_run_engine = Mock(return_value=sentinel_output)
|
||||
monkeypatch.setattr(llm, "_run_engine", _run_engine)
|
||||
|
||||
outputs = llm._run_chat(
|
||||
messages=messages,
|
||||
params=sampling_params,
|
||||
output_type=object,
|
||||
output_type=RequestOutput,
|
||||
use_tqdm=False,
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
)
|
||||
|
||||
assert outputs == sentinel_output
|
||||
assert llm._add_chat_requests.call_args.kwargs["mm_processor_kwargs"] == (
|
||||
assert _add_chat_requests.call_args.kwargs["mm_processor_kwargs"] == (
|
||||
mm_processor_kwargs
|
||||
)
|
||||
|
||||
|
||||
def test_run_completion_forwards_mm_processor_kwargs() -> None:
|
||||
def test_run_completion_forwards_mm_processor_kwargs(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
llm = _make_mock_llm()
|
||||
mm_processor_kwargs = {"min_pixels": 4 * 28 * 28}
|
||||
sampling_params = SamplingParams(max_tokens=1)
|
||||
sentinel_output = ["done"]
|
||||
|
||||
llm._add_completion_requests = Mock()
|
||||
llm._run_engine = Mock(return_value=sentinel_output)
|
||||
_add_completion_requests = Mock()
|
||||
monkeypatch.setattr(llm, "_add_completion_requests", _add_completion_requests)
|
||||
_run_engine = Mock(return_value=sentinel_output)
|
||||
monkeypatch.setattr(llm, "_run_engine", _run_engine)
|
||||
|
||||
outputs = llm._run_completion(
|
||||
prompts=["prompt"],
|
||||
params=sampling_params,
|
||||
output_type=object,
|
||||
output_type=RequestOutput,
|
||||
use_tqdm=False,
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
)
|
||||
|
||||
assert outputs == sentinel_output
|
||||
assert llm._add_completion_requests.call_args.kwargs["mm_processor_kwargs"] == (
|
||||
assert _add_completion_requests.call_args.kwargs["mm_processor_kwargs"] == (
|
||||
mm_processor_kwargs
|
||||
)
|
||||
|
||||
|
||||
def test_add_completion_requests_forwards_mm_processor_kwargs() -> None:
|
||||
def test_add_completion_requests_forwards_mm_processor_kwargs(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
llm = _make_mock_llm()
|
||||
mm_processor_kwargs = {"max_dynamic_patch": 4}
|
||||
sampling_params = SamplingParams(max_tokens=1)
|
||||
|
||||
llm._params_to_seq = Mock(return_value=[sampling_params])
|
||||
llm._lora_request_to_seq = Mock(return_value=[None])
|
||||
llm._priority_to_seq = Mock(return_value=[0])
|
||||
llm._preprocess_cmpl_one = Mock(return_value={"prompt_token_ids": [1]})
|
||||
_params_to_seq = Mock(return_value=[sampling_params])
|
||||
monkeypatch.setattr(llm, "_params_to_seq", _params_to_seq)
|
||||
_lora_request_to_seq = Mock(return_value=[None])
|
||||
monkeypatch.setattr(llm, "_lora_request_to_seq", _lora_request_to_seq)
|
||||
_priority_to_seq = Mock(return_value=[0])
|
||||
monkeypatch.setattr(llm, "_priority_to_seq", _priority_to_seq)
|
||||
_preprocess_cmpl_one = Mock(return_value={"prompt_token_ids": [1]})
|
||||
monkeypatch.setattr(llm, "_preprocess_cmpl_one", _preprocess_cmpl_one)
|
||||
|
||||
captured_prompts = []
|
||||
|
||||
@@ -160,7 +179,8 @@ def test_add_completion_requests_forwards_mm_processor_kwargs() -> None:
|
||||
captured_prompts.extend(prompts)
|
||||
return ["req-0"]
|
||||
|
||||
llm._render_and_add_requests = Mock(side_effect=fake_render_and_add_requests)
|
||||
_render_and_add_requests = Mock(side_effect=fake_render_and_add_requests)
|
||||
monkeypatch.setattr(llm, "_render_and_add_requests", _render_and_add_requests)
|
||||
|
||||
request_ids = llm._add_completion_requests(
|
||||
prompts=["prompt"],
|
||||
@@ -170,7 +190,7 @@ def test_add_completion_requests_forwards_mm_processor_kwargs() -> None:
|
||||
)
|
||||
|
||||
assert request_ids == ["req-0"]
|
||||
llm._preprocess_cmpl_one.assert_called_once_with(
|
||||
_preprocess_cmpl_one.assert_called_once_with(
|
||||
"prompt",
|
||||
None,
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
@@ -183,7 +203,7 @@ def test_preprocess_cmpl_applies_mm_processor_kwargs_to_renderer(
|
||||
) -> None:
|
||||
llm = _make_mock_llm()
|
||||
mm_processor_kwargs = {"num_crops": 8}
|
||||
prompt = {"prompt": "<image>", "multi_modal_data": {"image": object()}}
|
||||
prompt: TextPrompt = {"prompt": "<image>", "multi_modal_data": {"image": object()}}
|
||||
|
||||
renderer = Mock()
|
||||
renderer.default_cmpl_tok_params = Mock()
|
||||
@@ -213,7 +233,7 @@ def test_preprocess_cmpl_keeps_prompt_mm_processor_kwargs_when_no_override(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
llm = _make_mock_llm()
|
||||
prompt = {
|
||||
prompt: TextPrompt = {
|
||||
"prompt": "<image>",
|
||||
"multi_modal_data": {"image": object()},
|
||||
"mm_processor_kwargs": {"num_crops": 2},
|
||||
@@ -243,7 +263,9 @@ def test_preprocess_cmpl_keeps_prompt_mm_processor_kwargs_when_no_override(
|
||||
def test_preprocess_chat_applies_mm_processor_kwargs_to_renderer() -> None:
|
||||
llm = _make_mock_llm()
|
||||
mm_processor_kwargs = {"num_crops": 8}
|
||||
messages = [[{"role": "user", "content": "Describe this image."}]]
|
||||
messages: list[list[ChatCompletionMessageParam]] = [
|
||||
[{"role": "user", "content": "Describe this image."}]
|
||||
]
|
||||
|
||||
renderer = Mock()
|
||||
renderer.tokenizer = object()
|
||||
@@ -269,7 +291,9 @@ def test_preprocess_chat_applies_mm_processor_kwargs_to_renderer() -> None:
|
||||
|
||||
def test_preprocess_chat_omits_mm_processor_kwargs_when_no_override() -> None:
|
||||
llm = _make_mock_llm()
|
||||
messages = [[{"role": "user", "content": "Describe this image."}]]
|
||||
messages: list[list[ChatCompletionMessageParam]] = [
|
||||
[{"role": "user", "content": "Describe this image."}]
|
||||
]
|
||||
|
||||
renderer = Mock()
|
||||
renderer.tokenizer = object()
|
||||
@@ -291,7 +315,9 @@ def test_preprocess_chat_omits_mm_processor_kwargs_when_no_override() -> None:
|
||||
def test_preprocess_chat_defaults_add_special_tokens_to_false() -> None:
|
||||
# Matches `ChatCompletionRequest.add_special_tokens` on the server.
|
||||
llm = _make_mock_llm()
|
||||
messages = [[{"role": "user", "content": "hi"}]]
|
||||
messages: list[list[ChatCompletionMessageParam]] = [
|
||||
[{"role": "user", "content": "hi"}]
|
||||
]
|
||||
|
||||
renderer = Mock()
|
||||
renderer.tokenizer = object()
|
||||
@@ -309,7 +335,9 @@ def test_preprocess_chat_defaults_add_special_tokens_to_false() -> None:
|
||||
|
||||
def test_preprocess_chat_tokenization_kwargs_override_add_special_tokens() -> None:
|
||||
llm = _make_mock_llm()
|
||||
messages = [[{"role": "user", "content": "hi"}]]
|
||||
messages: list[list[ChatCompletionMessageParam]] = [
|
||||
[{"role": "user", "content": "hi"}]
|
||||
]
|
||||
|
||||
renderer = Mock()
|
||||
renderer.tokenizer = object()
|
||||
|
||||
@@ -47,18 +47,20 @@ async def client(server):
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def base64_encoded_audio() -> dict[str, str]:
|
||||
return {
|
||||
audio_url: encode_audio_base64(*fetch_audio(audio_url))
|
||||
for audio_url in TEST_AUDIO_URLS
|
||||
}
|
||||
encoded = {}
|
||||
for audio_url in TEST_AUDIO_URLS:
|
||||
audio, sample_rate = fetch_audio(audio_url)
|
||||
encoded[audio_url] = encode_audio_base64(audio, int(sample_rate))
|
||||
return encoded
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def url_encoded_audio() -> dict[str, str]:
|
||||
return {
|
||||
audio_url: encode_audio_url(*fetch_audio(audio_url))
|
||||
for audio_url in TEST_AUDIO_URLS
|
||||
}
|
||||
encoded = {}
|
||||
for audio_url in TEST_AUDIO_URLS:
|
||||
audio, sample_rate = fetch_audio(audio_url)
|
||||
encoded[audio_url] = encode_audio_url(audio, int(sample_rate))
|
||||
return encoded
|
||||
|
||||
|
||||
def dummy_messages_from_audio_url(
|
||||
|
||||
@@ -246,11 +246,15 @@ async def test_batched_chat_completions_logprob_token_ids(
|
||||
}
|
||||
|
||||
|
||||
def _make_request_output(prompt_idx: int, text: str) -> RequestOutput:
|
||||
def _make_request_output(
|
||||
prompt_idx: int,
|
||||
text: str,
|
||||
prompt_token_ids: list[int] | None = None,
|
||||
) -> RequestOutput:
|
||||
return RequestOutput(
|
||||
request_id=f"req-{prompt_idx}",
|
||||
prompt=None,
|
||||
prompt_token_ids=[1, 2, 3],
|
||||
prompt_token_ids=prompt_token_ids or [1, 2, 3],
|
||||
prompt_logprobs=None,
|
||||
outputs=[
|
||||
CompletionOutput(
|
||||
@@ -266,8 +270,12 @@ def _make_request_output(prompt_idx: int, text: str) -> RequestOutput:
|
||||
)
|
||||
|
||||
|
||||
async def _generator(prompt_idx: int, text: str) -> AsyncGenerator[RequestOutput, None]:
|
||||
yield _make_request_output(prompt_idx, text)
|
||||
async def _generator(
|
||||
prompt_idx: int,
|
||||
text: str,
|
||||
prompt_token_ids: list[int] | None = None,
|
||||
) -> AsyncGenerator[RequestOutput, None]:
|
||||
yield _make_request_output(prompt_idx, text, prompt_token_ids)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -402,3 +410,47 @@ def test_batch_rejects_kv_transfer_prompt_token_ids():
|
||||
],
|
||||
kv_transfer_params={"prompt_token_ids": [10, 20, 30]},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skip_global_cleanup
|
||||
async def test_batched_parser_receives_each_prompt_prefix() -> None:
|
||||
prefixes = []
|
||||
|
||||
class RecordingParser:
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.prompt_token_ids = None
|
||||
|
||||
def set_prompt_token_ids(self, prompt_token_ids):
|
||||
self.prompt_token_ids = prompt_token_ids
|
||||
|
||||
def parse(self, model_output, **kwargs):
|
||||
prefixes.append(self.prompt_token_ids)
|
||||
return None, model_output, None
|
||||
|
||||
serving = OpenAIServingChatBatch.__new__(OpenAIServingChatBatch)
|
||||
serving.response_role = "assistant"
|
||||
serving.system_fingerprint = None
|
||||
serving.chat_template = None
|
||||
serving.chat_template_content_format = "auto"
|
||||
serving.default_chat_template_kwargs = {}
|
||||
|
||||
request = BatchChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[
|
||||
[{"role": "user", "content": "first"}],
|
||||
[{"role": "user", "content": "second"}],
|
||||
],
|
||||
)
|
||||
await serving.chat_completion_full_generator_batch(
|
||||
request=request,
|
||||
generators=[_generator(0, "one", [1]), _generator(1, "two", [2])],
|
||||
request_id="req-prefix",
|
||||
model_name="test-model",
|
||||
all_conversations=request.messages,
|
||||
tokenizer=object(),
|
||||
request_metadata=RequestResponseMetadata(request_id="req-prefix"),
|
||||
parser_cls=RecordingParser,
|
||||
)
|
||||
|
||||
assert prefixes == [[1], [2]]
|
||||
|
||||
@@ -564,6 +564,9 @@ class MockModelConfig:
|
||||
def get_diff_sampling_param(self):
|
||||
return self.diff_sampling_param or {}
|
||||
|
||||
def get_vocab_size(self) -> int:
|
||||
return 50257
|
||||
|
||||
|
||||
@dataclass
|
||||
class MockParallelConfig:
|
||||
@@ -787,6 +790,64 @@ async def test_chat_per_request_metrics_suppressed_for_n_greater_than_one():
|
||||
assert response.metrics is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_full_generator_gives_each_choice_parser_the_prompt():
|
||||
prompts: list[list[int] | None] = []
|
||||
|
||||
class RecordingParser:
|
||||
tool_parser_cls = None
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.prompt_token_ids = None
|
||||
|
||||
def set_prompt_token_ids(self, prompt_token_ids):
|
||||
self.prompt_token_ids = list(prompt_token_ids)
|
||||
|
||||
def parse(self, model_output, request, **kwargs):
|
||||
prompts.append(self.prompt_token_ids)
|
||||
return None, model_output, None
|
||||
|
||||
def count_reasoning_tokens(self, token_ids):
|
||||
return 0
|
||||
|
||||
serving = _build_minimal_metrics_serving_chat(enable_per_request_metrics=False)
|
||||
serving.parser_cls = RecordingParser
|
||||
serving.model_config = None
|
||||
serving.chat_template = None
|
||||
serving.chat_template_content_format = "auto"
|
||||
serving.default_chat_template_kwargs = {}
|
||||
request_output = _make_metrics_request_output()
|
||||
request_output.prompt_token_ids = [7, 8]
|
||||
request_output.outputs.append(
|
||||
CompletionOutput(
|
||||
index=1,
|
||||
text="Hi",
|
||||
token_ids=[102],
|
||||
cumulative_logprob=None,
|
||||
logprobs=None,
|
||||
finish_reason="stop",
|
||||
)
|
||||
)
|
||||
|
||||
await serving.chat_completion_full_generator(
|
||||
ChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[{"role": "user", "content": "Test prompt"}],
|
||||
max_tokens=10,
|
||||
n=2,
|
||||
),
|
||||
_single_request_output(request_output),
|
||||
"chatcmpl-test-id",
|
||||
"test-model",
|
||||
conversation=[{"role": "user", "content": "Test"}],
|
||||
tokenizer=MagicMock(),
|
||||
request_metadata=RequestResponseMetadata(request_id="chatcmpl-test-id"),
|
||||
parser=RecordingParser(),
|
||||
)
|
||||
|
||||
assert prompts == [[7, 8], [7, 8]]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_streaming_metrics_ride_on_usage_chunk():
|
||||
serving = _build_minimal_metrics_serving_chat(enable_per_request_metrics=True)
|
||||
@@ -1605,6 +1666,21 @@ async def test_serving_chat_truncation_side_controls_prompt_truncation():
|
||||
assert left_token_ids == full_token_ids[-4:]
|
||||
|
||||
|
||||
def test_serving_chat_starts_without_tokenizer():
|
||||
"""Regression test: building the serving objects must not require a
|
||||
tokenizer, which is absent under `skip_tokenizer_init`."""
|
||||
mock_engine = _build_mock_engine()
|
||||
mock_engine.model_config = MockModelConfig(skip_tokenizer_init=True)
|
||||
mock_engine.renderer = HfRenderer(
|
||||
MockVllmConfig(mock_engine.model_config, parallel_config=MockParallelConfig()),
|
||||
None,
|
||||
)
|
||||
|
||||
serving_chat = _build_serving_chat(mock_engine)
|
||||
|
||||
assert serving_chat.parser_cls is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_serving_chat_mistral_token_ids_prompt_is_validated():
|
||||
"""Regression test: when the Mistral tokenizer path returns token IDs
|
||||
@@ -2860,7 +2936,16 @@ async def test_chat_kv_transfer_prompt_token_ids_rejects_invalid_ids(ids):
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_kv_transfer_prompt_token_ids_rejection_notifies_kv_connector():
|
||||
@pytest.mark.parametrize(
|
||||
("ids", "stream", "match"),
|
||||
[
|
||||
([1.5], False, "non-negative integers"),
|
||||
([999999], True, r"out of vocabulary.*kv_transfer_params\.prompt_token_ids"),
|
||||
],
|
||||
)
|
||||
async def test_chat_kv_transfer_prompt_token_ids_rejection_notifies_kv_connector(
|
||||
ids, stream, match
|
||||
):
|
||||
"""A render-time 400 on a decode request notifies the KV connector."""
|
||||
mock_engine = _build_mock_engine()
|
||||
mock_engine.notify_kv_transfer_request_rejected = AsyncMock()
|
||||
@@ -2870,9 +2955,10 @@ async def test_chat_kv_transfer_prompt_token_ids_rejection_notifies_kv_connector
|
||||
request = ChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
kv_transfer_params={"prompt_token_ids": [1.5], "do_remote_prefill": True},
|
||||
stream=stream,
|
||||
kv_transfer_params={"prompt_token_ids": ids, "do_remote_prefill": True},
|
||||
)
|
||||
with pytest.raises(VLLMValidationError, match="non-negative integers"):
|
||||
with pytest.raises(VLLMValidationError, match=match):
|
||||
await serving_chat.create_chat_completion(request)
|
||||
|
||||
mock_engine.notify_kv_transfer_request_rejected.assert_awaited_once_with(
|
||||
|
||||
@@ -37,7 +37,6 @@ def pairs_of_event_types() -> dict[str, str]:
|
||||
"response.content_part.done": "response.content_part.added",
|
||||
"response.output_text.done": "response.output_text.delta",
|
||||
"response.reasoning_text.done": "response.reasoning_text.delta",
|
||||
"response.reasoning_part.done": "response.reasoning_part.added",
|
||||
"response.mcp_call_arguments.done": "response.mcp_call_arguments.delta",
|
||||
"response.mcp_call.completed": "response.mcp_call.in_progress",
|
||||
"response.function_call_arguments.done": "response.function_call_arguments.delta", # noqa: E501
|
||||
@@ -126,6 +125,11 @@ def _validate_event_pairing(events: list, pairs_of_event_types: dict[str, str])
|
||||
etype = event.type
|
||||
if etype in end_events:
|
||||
expected_start = pairs_of_event_types[etype]
|
||||
# Deltas are optional, e.g. a function call with no arguments.
|
||||
if expected_start.endswith(".delta") and (
|
||||
not stack or stack[-1] != expected_start
|
||||
):
|
||||
continue
|
||||
assert stack and stack[-1] == expected_start, (
|
||||
f"Stack mismatch for {etype}: "
|
||||
f"expected {expected_start}, "
|
||||
@@ -245,11 +249,8 @@ def _validate_field_consistency(events: list) -> None:
|
||||
active_content_index = None
|
||||
continue
|
||||
|
||||
# --- content_part / reasoning_part added: sets content_index
|
||||
if etype in (
|
||||
"response.content_part.added",
|
||||
"response.reasoning_part.added",
|
||||
):
|
||||
# --- content_part added: sets content_index
|
||||
if etype == "response.content_part.added":
|
||||
_assert_item_fields(event, etype, active_item_id, active_output_index)
|
||||
content_index = getattr(event, "content_index", None)
|
||||
if active_content_index is None:
|
||||
|
||||
@@ -340,12 +340,12 @@ async def test_function_calling_with_streaming_types(
|
||||
# response.created
|
||||
# -> response.in_progress
|
||||
# -> response.output_item.added
|
||||
# -> response.reasoning_part.added
|
||||
# -> response.content_part.added
|
||||
# -> response.reasoning_text.delta
|
||||
# ....
|
||||
# -> response.reasoning_text.delta
|
||||
# -> response.reasoning_text.done
|
||||
# -> response.reasoning_part.done
|
||||
# -> response.content_part.done
|
||||
# -> response.output_item.done
|
||||
# -> response.output_item.added
|
||||
# -> response.content_part.added
|
||||
@@ -368,12 +368,12 @@ async def test_function_calling_with_streaming_types(
|
||||
# response.created
|
||||
# -> response.in_progress
|
||||
# -> response.output_item.added
|
||||
# -> response.reasoning_part.added
|
||||
# -> response.content_part.added
|
||||
# -> response.reasoning_text.delta
|
||||
# ....
|
||||
# -> response.reasoning_text.delta
|
||||
# -> response.reasoning_text.done
|
||||
# -> response.reasoning_part.done
|
||||
# -> response.content_part.done
|
||||
# -> response.output_item.done
|
||||
# -> response.output_item.added
|
||||
# -> response.content_part.added
|
||||
@@ -424,7 +424,6 @@ async def test_function_calling_with_streaming_types(
|
||||
"response.output_text.done": "response.output_text.delta",
|
||||
"response.content_part.done": "response.content_part.added",
|
||||
"response.reasoning_text.done": "response.reasoning_text.delta",
|
||||
"response.reasoning_part.done": "response.reasoning_part.added",
|
||||
"response.function_call_arguments.done": "response.function_call_arguments.delta", # noqa
|
||||
}
|
||||
|
||||
|
||||
@@ -467,10 +467,7 @@ async def test_streaming(client: OpenAI, model_name: str, background: bool):
|
||||
assert event.item_id == current_item_id
|
||||
|
||||
# Verify content indices
|
||||
if event.type in [
|
||||
"response.content_part.added",
|
||||
"response.reasoning_part.added",
|
||||
]:
|
||||
if event.type == "response.content_part.added":
|
||||
assert event.content_index == current_content_index + 1
|
||||
current_content_index = event.content_index
|
||||
elif event.type in [
|
||||
|
||||
@@ -6,10 +6,13 @@ These tests verify that ParsableContext correctly delegates to the unified
|
||||
Parser (via parse) and properly builds response output items.
|
||||
"""
|
||||
|
||||
import json
|
||||
from collections.abc import Sequence
|
||||
from unittest.mock import MagicMock
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
from openai.types.responses import ResponseFunctionToolCall
|
||||
|
||||
from vllm.entrypoints.generate.base.protocol import (
|
||||
DeltaMessage,
|
||||
@@ -17,6 +20,7 @@ from vllm.entrypoints.generate.base.protocol import (
|
||||
FunctionCall,
|
||||
ToolCall,
|
||||
)
|
||||
from vllm.entrypoints.mcp.tool import HarmonyPythonTool
|
||||
from vllm.entrypoints.openai.responses.context import ParsableContext
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.outputs import CompletionOutput, RequestOutput
|
||||
@@ -317,6 +321,65 @@ def test_process_extracts_tool_calls():
|
||||
assert tool_item.status == "completed"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("tool_name", "session_name", "dispatched_name", "arguments"),
|
||||
[
|
||||
("code_interpreter", "python", "python", {"code": "print(42)"}),
|
||||
("web_search_preview", "browser", "search", {"query": "vLLM"}),
|
||||
("container.exec", "container", "exec", {"cmd": ["pwd"]}),
|
||||
],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_builtin_tool_output_preserves_function_call_id(
|
||||
tool_name, session_name, dispatched_name, arguments
|
||||
):
|
||||
"""Built-in tool outputs remain correlated with their originating call."""
|
||||
tool_session = MagicMock()
|
||||
tool_session.call_tool = AsyncMock(
|
||||
return_value=SimpleNamespace(content=[SimpleNamespace(text="result")])
|
||||
)
|
||||
context = _make_context(None, available_tools=[session_name])
|
||||
tool_call = ResponseFunctionToolCall(
|
||||
id=f"fc_{session_name}",
|
||||
call_id=f"call_{session_name}",
|
||||
type="function_call",
|
||||
name=tool_name,
|
||||
arguments=json.dumps(arguments),
|
||||
)
|
||||
context.response_messages.append(tool_call)
|
||||
context._tool_sessions[session_name] = tool_session
|
||||
|
||||
output = await context.call_tool()
|
||||
|
||||
tool_session.call_tool.assert_awaited_once_with(dispatched_name, arguments)
|
||||
assert output[0].call_id == tool_call.call_id
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_local_python_tool_output_preserves_function_call_id():
|
||||
"""Local code-interpreter output remains correlated with its call."""
|
||||
|
||||
async def process(_):
|
||||
yield SimpleNamespace(content=[SimpleNamespace(text="result")])
|
||||
|
||||
python_tool = object.__new__(HarmonyPythonTool)
|
||||
python_tool.python_tool = MagicMock(process=process)
|
||||
context = _make_context(None, available_tools=["python"])
|
||||
tool_call = ResponseFunctionToolCall(
|
||||
id="fc_python",
|
||||
call_id="call_python",
|
||||
type="function_call",
|
||||
name="code_interpreter",
|
||||
arguments='{"code": "print(42)"}',
|
||||
)
|
||||
context.response_messages.append(tool_call)
|
||||
context._tool_sessions["python"] = python_tool
|
||||
|
||||
output = await context.call_tool()
|
||||
|
||||
assert output[0].call_id == tool_call.call_id
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: finish_reason tracking
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -34,6 +34,7 @@ from openai_harmony import Message as OpenAIHarmonyMessage
|
||||
from openai_harmony import Role
|
||||
|
||||
import vllm.envs as envs
|
||||
from tests.entrypoints.openai.chat_completion.test_serving_chat import MockModelConfig
|
||||
from vllm.entrypoints.generate.base.protocol import (
|
||||
DeltaFunctionCall,
|
||||
DeltaMessage,
|
||||
@@ -64,6 +65,7 @@ from vllm.entrypoints.openai.responses.streaming_events import (
|
||||
)
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.inputs import tokens_input
|
||||
from vllm.logprobs import Logprob as SampleLogprob
|
||||
from vllm.outputs import CompletionOutput, RequestOutput
|
||||
from vllm.parser.harmony import Segment
|
||||
from vllm.renderers import TokenizeParams
|
||||
@@ -126,14 +128,6 @@ def test_serialize_message_pydantic_model_returns_dict() -> None:
|
||||
assert serialized["message"] == "hello"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_serving_responses():
|
||||
"""Create a mock OpenAIServingResponses instance."""
|
||||
serving_responses = MagicMock(spec=OpenAIServingResponses)
|
||||
serving_responses.tool_server = MagicMock(spec=ToolServer)
|
||||
return serving_responses
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_context():
|
||||
"""Create a mock conversation context."""
|
||||
@@ -301,6 +295,102 @@ async def test_online_renderer_renders_non_harmony_responses_with_explicit_histo
|
||||
assert preprocess_kwargs["default_template_kwargs"]["request_default"] == "kept"
|
||||
|
||||
|
||||
_REUSED_IDS = [10, 20, 30]
|
||||
_IMAGE_PART = {
|
||||
"type": "input_image",
|
||||
"image_url": "https://example.com/a.png",
|
||||
"detail": "auto",
|
||||
}
|
||||
|
||||
|
||||
async def _render_responses_with_reuse(request, previous_messages=None):
|
||||
online_renderer = OnlineRenderer(
|
||||
model_config=MockModelConfig(),
|
||||
renderer=MagicMock(),
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
)
|
||||
render_chat = AsyncMock(return_value=([[]], [tokens_input([1, 2])]))
|
||||
online_renderer.renderer.render_chat_async = render_chat
|
||||
|
||||
result = await online_renderer.render_responses(
|
||||
request, previous_messages=previous_messages
|
||||
)
|
||||
|
||||
assert not isinstance(result, ErrorResponse)
|
||||
return render_chat, result.engine_input
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("ids", [_REUSED_IDS, [1.5]])
|
||||
@pytest.mark.parametrize(
|
||||
"request_input",
|
||||
[
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "input_text", "text": "look"}, _IMAGE_PART],
|
||||
}
|
||||
],
|
||||
[
|
||||
{
|
||||
"type": "function_call",
|
||||
"call_id": "call_1",
|
||||
"name": "shot",
|
||||
"arguments": "{}",
|
||||
},
|
||||
{
|
||||
"type": "function_call_output",
|
||||
"call_id": "call_1",
|
||||
"output": [_IMAGE_PART],
|
||||
},
|
||||
],
|
||||
],
|
||||
ids=["input_image", "tool_output_image"],
|
||||
)
|
||||
async def test_responses_kv_transfer_prompt_token_ids_ignored_with_non_text_input(
|
||||
request_input, ids
|
||||
):
|
||||
request = ResponsesRequest(
|
||||
input=request_input,
|
||||
kv_transfer_params={"do_remote_prefill": True, "prompt_token_ids": ids},
|
||||
)
|
||||
|
||||
render_chat, _ = await _render_responses_with_reuse(request)
|
||||
|
||||
render_chat.assert_awaited_once()
|
||||
assert request.kv_transfer_params == {"do_remote_prefill": True}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_kv_transfer_prompt_token_ids_ignored_with_media_in_history():
|
||||
request = ResponsesRequest(
|
||||
input="next", kv_transfer_params={"prompt_token_ids": _REUSED_IDS}
|
||||
)
|
||||
|
||||
render_chat, _ = await _render_responses_with_reuse(
|
||||
request, previous_messages=[{"role": "user", "content": [_IMAGE_PART]}]
|
||||
)
|
||||
|
||||
render_chat.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_kv_transfer_prompt_token_ids_kept_with_text_input():
|
||||
request = ResponsesRequest(
|
||||
input=[
|
||||
{"role": "user", "content": [{"type": "input_text", "text": "hi"}]},
|
||||
{"role": "assistant", "content": "ok"},
|
||||
],
|
||||
kv_transfer_params={"prompt_token_ids": _REUSED_IDS},
|
||||
)
|
||||
|
||||
_, engine_input = await _render_responses_with_reuse(request)
|
||||
|
||||
assert engine_input["prompt_token_ids"] == _REUSED_IDS
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_online_renderer_rejects_untrusted_responses_chat_template():
|
||||
renderer = _new_online_renderer()
|
||||
@@ -1359,6 +1449,7 @@ def _make_simple_context_with_output(
|
||||
def _make_serving_instance(
|
||||
*,
|
||||
reasoning_parser: str = "",
|
||||
tool_parser: str | None = None,
|
||||
enable_per_request_metrics: bool = False,
|
||||
) -> OpenAIServingResponses:
|
||||
engine_client = MagicMock()
|
||||
@@ -1379,6 +1470,8 @@ def _make_serving_instance(
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
reasoning_parser=reasoning_parser,
|
||||
enable_auto_tools=tool_parser is not None,
|
||||
tool_parser=tool_parser,
|
||||
enable_per_request_metrics=enable_per_request_metrics,
|
||||
)
|
||||
|
||||
@@ -1977,3 +2070,113 @@ class TestAutoToolStreaming:
|
||||
assert len(function_done) == 1
|
||||
assert function_done[0].item.name == "get_weather"
|
||||
assert function_done[0].item.arguments == tool_args
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_completed_response_reuses_streamed_items(monkeypatch):
|
||||
"""response.completed must carry the streamed items (same ids,
|
||||
call_id, logprobs) rather than a reparse of the full output."""
|
||||
monkeypatch.setattr(envs, "VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT", False)
|
||||
serving = _make_serving_instance_with_reasoning()
|
||||
call = DeltaToolCall(
|
||||
id="chatcmpl-tool-parser-id",
|
||||
index=0,
|
||||
function=DeltaFunctionCall(name="get_weather", arguments="{}"),
|
||||
)
|
||||
response_parser = _mock_parser_with_reasoning(
|
||||
serving,
|
||||
[
|
||||
DeltaMessage(reasoning="think"),
|
||||
DeltaMessage(content="Hi"),
|
||||
DeltaMessage(tool_calls=[call]),
|
||||
],
|
||||
)
|
||||
response_parser.count_reasoning_tokens = MagicMock(return_value=1)
|
||||
context = SimpleContext(response_parser=response_parser)
|
||||
|
||||
async def result_generator():
|
||||
for token_id, text in [(10, "think"), (20, "Hi"), (30, "call")]:
|
||||
output = _make_request_output(text, [token_id])
|
||||
output.outputs[0].logprobs = [
|
||||
{token_id: SampleLogprob(logprob=-0.5, decoded_token=text)}
|
||||
]
|
||||
context.append_output(output)
|
||||
yield context
|
||||
|
||||
events = [
|
||||
event
|
||||
async for event in serving.responses_stream_generator(
|
||||
request=ResponsesRequest(
|
||||
input="hi",
|
||||
include=["message.output_text.logprobs"],
|
||||
stream=True,
|
||||
store=False,
|
||||
),
|
||||
sampling_params=SamplingParams(max_tokens=16),
|
||||
result_generator=result_generator(),
|
||||
context=context,
|
||||
model_name="test-model",
|
||||
tokenizer=MagicMock(),
|
||||
request_metadata=RequestResponseMetadata(request_id="req"),
|
||||
)
|
||||
]
|
||||
|
||||
streamed = [e.item for e in events if e.type == "response.output_item.done"]
|
||||
assert events[-1].response.output == streamed
|
||||
_, message, function_call = streamed
|
||||
assert [lp.token for lp in message.content[0].logprobs] == ["Hi"]
|
||||
assert function_call.call_id == "chatcmpl-tool-parser-id"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(("parallel_tool_calls", "num_calls"), [(False, 1), (None, 3)])
|
||||
async def test_streaming_parallel_tool_calls(
|
||||
monkeypatch, parallel_tool_calls, num_calls
|
||||
):
|
||||
"""parallel_tool_calls=False keeps only the first call even when the grammar
|
||||
does not limit the model; None keeps every call."""
|
||||
monkeypatch.setattr(envs, "VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT", False)
|
||||
serving = _make_serving_instance(tool_parser="hermes")
|
||||
request = ResponsesRequest(
|
||||
input="hi",
|
||||
tools=[{"type": "function", "name": "get_weather", "parameters": {}}],
|
||||
parallel_tool_calls=parallel_tool_calls,
|
||||
stream=True,
|
||||
)
|
||||
context = SimpleContext(
|
||||
response_parser=serving._make_response_parser(request, MagicMock(), {})
|
||||
)
|
||||
args = [f'{{"city": "{city}"}}' for city in ("Paris", "Berlin", "Tokyo")]
|
||||
|
||||
async def result_generator():
|
||||
for i, arg in enumerate(args):
|
||||
text = f'<tool_call>\n{{"name": "get_weather", "arguments": {arg}}}'
|
||||
context.append_output(_make_request_output(f"{text}\n</tool_call>\n", [i]))
|
||||
yield context
|
||||
|
||||
events = [
|
||||
event
|
||||
async for event in serving.responses_stream_generator(
|
||||
request=request,
|
||||
sampling_params=SamplingParams(max_tokens=64),
|
||||
result_generator=result_generator(),
|
||||
context=context,
|
||||
model_name="test-model",
|
||||
tokenizer=MagicMock(),
|
||||
request_metadata=RequestResponseMetadata(request_id="req"),
|
||||
)
|
||||
]
|
||||
|
||||
assert [
|
||||
event.output_index
|
||||
for event in events
|
||||
if event.type == "response.output_item.added"
|
||||
] == list(range(num_calls))
|
||||
assert all(getattr(event, "output_index", 0) < num_calls for event in events)
|
||||
assert [
|
||||
event.item.arguments
|
||||
for event in events
|
||||
if event.type == "response.output_item.done"
|
||||
] == args[:num_calls]
|
||||
assert [item.arguments for item in events[-1].response.output] == args[:num_calls]
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from openai.types.responses import ResponseFunctionWebSearch
|
||||
import pytest
|
||||
from openai.types.responses import ResponseFunctionWebSearch, ResponseStreamEvent
|
||||
from openai_harmony import Message, Role
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from vllm.entrypoints.generate.base.protocol import (
|
||||
DeltaFunctionCall,
|
||||
@@ -14,6 +16,8 @@ from vllm.entrypoints.openai.responses.streaming_events import (
|
||||
StreamingState,
|
||||
_StateType,
|
||||
emit_browser_tool_events,
|
||||
emit_reasoning_delta_events,
|
||||
emit_reasoning_done_events,
|
||||
split_delta,
|
||||
)
|
||||
|
||||
@@ -40,6 +44,41 @@ def test_browser_find_uses_responses_action_type():
|
||||
assert event.item.action.url == "cursor:42"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("harmony", [False, True], ids=["simple", "harmony"])
|
||||
def test_reasoning_content_parts_use_sdk_events(harmony):
|
||||
text = "Let me think."
|
||||
if harmony:
|
||||
state = StreamingState()
|
||||
events = emit_reasoning_delta_events(text, state)
|
||||
events.extend(emit_reasoning_done_events(text, state))
|
||||
else:
|
||||
processor = SimpleStreamingEventProcessor()
|
||||
events = _run_through_processor(processor, DeltaMessage(reasoning=text))
|
||||
events.extend(processor.close_current())
|
||||
|
||||
assert [event.type for event in events] == [
|
||||
"response.output_item.added",
|
||||
"response.content_part.added",
|
||||
"response.reasoning_text.delta",
|
||||
"response.reasoning_text.done",
|
||||
"response.content_part.done",
|
||||
"response.output_item.done",
|
||||
]
|
||||
added, part_added, delta, text_done, part_done, done = events
|
||||
assert part_added.part.type == part_done.part.type == "reasoning_text"
|
||||
assert part_added.part.text == ""
|
||||
assert delta.delta == text_done.text == part_done.part.text == text
|
||||
assert done.item.content[0].text == text
|
||||
assert done.item.id == added.item.id
|
||||
for event in events[1:-1]:
|
||||
assert event.item_id == added.item.id
|
||||
assert event.content_index == part_added.content_index
|
||||
validator = TypeAdapter(ResponseStreamEvent)
|
||||
for event in events:
|
||||
assert event.output_index == added.output_index
|
||||
validator.validate_python(event.model_dump())
|
||||
|
||||
|
||||
def _make_tool_call(
|
||||
index: int, name: str | None = None, arguments: str | None = None
|
||||
) -> DeltaToolCall:
|
||||
|
||||
@@ -225,7 +225,7 @@ def online_renderer(model_config: MockModelConfig, request) -> OnlineRenderer:
|
||||
renderer.tokenizer = MagicMock()
|
||||
|
||||
return OnlineRenderer(
|
||||
model_config=model_config, # type: ignore[arg-type]
|
||||
model_config=model_config,
|
||||
renderer=renderer,
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
|
||||
@@ -7,6 +7,7 @@ import torch
|
||||
|
||||
from tests.models.utils import softmax
|
||||
from vllm import LLM, ClassificationRequestOutput, PoolingParams
|
||||
from vllm.inputs import PromptType
|
||||
from vllm.tasks import PoolingTask
|
||||
|
||||
MODEL_NAME = "jason9693/Qwen2.5-1.5B-apeach"
|
||||
@@ -53,7 +54,8 @@ def test_token_ids_prompts(llm: LLM):
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_list_prompts(llm: LLM):
|
||||
outputs = llm.classify([prompt, prompt_token_ids], use_tqdm=False)
|
||||
prompts: list[PromptType] = [prompt, prompt_token_ids]
|
||||
outputs = llm.classify(prompts, use_tqdm=False)
|
||||
assert len(outputs) == 2
|
||||
for i in range(len(outputs)):
|
||||
assert isinstance(outputs[i], ClassificationRequestOutput)
|
||||
|
||||
@@ -21,6 +21,7 @@ from vllm.entrypoints.pooling.typing import (
|
||||
PoolingEngineInput,
|
||||
PoolingServeContext,
|
||||
)
|
||||
from vllm.inputs import tokens_input
|
||||
from vllm.outputs import PoolingOutput, PoolingRequestOutput
|
||||
|
||||
|
||||
@@ -28,7 +29,7 @@ class TestEmbeddingRequestParsing:
|
||||
"""Unit tests for OpenAI embedding request parsing."""
|
||||
|
||||
def test_input_messages_parses_as_chat_request(self):
|
||||
request = TypeAdapter(EmbeddingRequest).validate_python(
|
||||
request: EmbeddingRequest = TypeAdapter(EmbeddingRequest).validate_python(
|
||||
{
|
||||
"model": "test",
|
||||
"input": [{"role": "user", "content": "hello"}],
|
||||
@@ -42,7 +43,7 @@ class TestEmbeddingRequestParsing:
|
||||
assert request.chat_template_kwargs == {"instruction": "Represent the query: "}
|
||||
|
||||
def test_batched_input_messages_parses_as_batch_chat_input_request(self):
|
||||
request = TypeAdapter(EmbeddingRequest).validate_python(
|
||||
request: EmbeddingRequest = TypeAdapter(EmbeddingRequest).validate_python(
|
||||
{
|
||||
"model": "test",
|
||||
"input": [
|
||||
@@ -65,7 +66,7 @@ class TestEmbeddingRequestParsing:
|
||||
assert request.chat_template_kwargs == {"instruction": "Represent the query: "}
|
||||
|
||||
def test_token_ids_still_parse_as_completion_request(self):
|
||||
request = TypeAdapter(EmbeddingRequest).validate_python(
|
||||
request: EmbeddingRequest = TypeAdapter(EmbeddingRequest).validate_python(
|
||||
{
|
||||
"model": "test",
|
||||
"input": [[1, 2, 3], [4, 5]],
|
||||
@@ -76,7 +77,7 @@ class TestEmbeddingRequestParsing:
|
||||
assert request.input == [[1, 2, 3], [4, 5]]
|
||||
|
||||
def test_messages_still_parses_as_chat_request(self):
|
||||
request = TypeAdapter(EmbeddingRequest).validate_python(
|
||||
request: EmbeddingRequest = TypeAdapter(EmbeddingRequest).validate_python(
|
||||
{
|
||||
"model": "test",
|
||||
"messages": [{"role": "user", "content": "hello"}],
|
||||
@@ -89,7 +90,7 @@ class TestEmbeddingRequestParsing:
|
||||
assert request.chat_template_kwargs == {"instruction": "Represent the query: "}
|
||||
|
||||
def test_batched_messages_parses_as_batch_chat_request(self):
|
||||
request = TypeAdapter(EmbeddingRequest).validate_python(
|
||||
request: EmbeddingRequest = TypeAdapter(EmbeddingRequest).validate_python(
|
||||
{
|
||||
"model": "test",
|
||||
"messages": [
|
||||
@@ -416,7 +417,7 @@ class TestChunkedEmbeddingProcessing:
|
||||
|
||||
@staticmethod
|
||||
def _make_context() -> PoolingServeContext[EmbeddingCompletionRequest]:
|
||||
request = TypeAdapter(EmbeddingRequest).validate_python(
|
||||
request: EmbeddingRequest = TypeAdapter(EmbeddingRequest).validate_python(
|
||||
{
|
||||
"model": "test",
|
||||
"input": [[0, 1, 2, 3, 4], [10, 11]],
|
||||
@@ -431,13 +432,13 @@ class TestChunkedEmbeddingProcessing:
|
||||
request_id="embd-client-prompt-999-chunk-888",
|
||||
engine_inputs=[
|
||||
PoolingEngineInput(
|
||||
prompts={"prompt_token_ids": [0, 1, 2, 3, 4]},
|
||||
prompts=tokens_input(prompt_token_ids=[0, 1, 2, 3, 4]),
|
||||
params=pooling_params,
|
||||
lora_requests=None,
|
||||
priorities=0,
|
||||
),
|
||||
PoolingEngineInput(
|
||||
prompts={"prompt_token_ids": [10, 11]},
|
||||
prompts=tokens_input(prompt_token_ids=[10, 11]),
|
||||
params=pooling_params,
|
||||
lora_requests=None,
|
||||
priorities=0,
|
||||
|
||||
@@ -7,6 +7,7 @@ import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm import LLM, EmbeddingRequestOutput, PoolingParams
|
||||
from vllm.inputs import PromptType
|
||||
from vllm.tasks import PoolingTask
|
||||
|
||||
MODEL_NAME = "intfloat/multilingual-e5-small"
|
||||
@@ -54,7 +55,8 @@ def test_token_ids_prompts(llm: LLM):
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_list_prompts(llm: LLM):
|
||||
outputs = llm.embed([prompt, prompt_token_ids], use_tqdm=False)
|
||||
prompts: list[PromptType] = [prompt, prompt_token_ids]
|
||||
outputs = llm.embed(prompts, use_tqdm=False)
|
||||
assert len(outputs) == 2
|
||||
for i in range(len(outputs)):
|
||||
assert isinstance(outputs[i], EmbeddingRequestOutput)
|
||||
|
||||
@@ -12,9 +12,13 @@ import requests
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from tests.models.language.pooling.embed_utils import run_embedding_correctness_test
|
||||
from tests.models.language.pooling.embed_utils import (
|
||||
float_embeddings,
|
||||
run_embedding_correctness_test,
|
||||
)
|
||||
from tests.models.utils import check_embeddings_close
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
|
||||
from vllm.entrypoints.pooling.embed.protocol import EmbeddingResponse
|
||||
from vllm.entrypoints.pooling.pooling.protocol import PoolingResponse
|
||||
from vllm.entrypoints.pooling.utils import (
|
||||
@@ -145,7 +149,7 @@ async def test_completion_request(
|
||||
assert embeddings.usage.prompt_tokens == len(input_tokens)
|
||||
assert embeddings.usage.total_tokens == len(input_tokens)
|
||||
|
||||
vllm_outputs = [d.embedding for d in embeddings.data]
|
||||
vllm_outputs = float_embeddings(embeddings)
|
||||
run_embedding_correctness_test(hf_model, [input_text], vllm_outputs)
|
||||
|
||||
# test input: list[int]
|
||||
@@ -165,7 +169,7 @@ async def test_completion_request(
|
||||
assert embeddings.usage.prompt_tokens == len(input_tokens)
|
||||
assert embeddings.usage.total_tokens == len(input_tokens)
|
||||
|
||||
vllm_outputs = [d.embedding for d in embeddings.data]
|
||||
vllm_outputs = float_embeddings(embeddings)
|
||||
run_embedding_correctness_test(hf_model, [input_text], vllm_outputs)
|
||||
|
||||
|
||||
@@ -194,7 +198,7 @@ async def test_completion_request_batched(
|
||||
assert embeddings.usage.prompt_tokens == len(input_tokens) * N
|
||||
assert embeddings.usage.total_tokens == len(input_tokens) * N
|
||||
|
||||
vllm_outputs = [d.embedding for d in embeddings.data]
|
||||
vllm_outputs = float_embeddings(embeddings)
|
||||
run_embedding_correctness_test(hf_model, input_texts, vllm_outputs)
|
||||
|
||||
# test list[list[int]]
|
||||
@@ -214,7 +218,7 @@ async def test_completion_request_batched(
|
||||
assert embeddings.usage.prompt_tokens == len(input_tokens) * N
|
||||
assert embeddings.usage.total_tokens == len(input_tokens) * N
|
||||
|
||||
vllm_outputs = [d.embedding for d in embeddings.data]
|
||||
vllm_outputs = float_embeddings(embeddings)
|
||||
run_embedding_correctness_test(hf_model, input_texts, vllm_outputs)
|
||||
|
||||
|
||||
@@ -331,7 +335,7 @@ async def test_padding(client: openai.AsyncOpenAI, model_name: str):
|
||||
async def test_chat_request(
|
||||
server: RemoteOpenAIServer, client: openai.AsyncOpenAI, model_name: str
|
||||
):
|
||||
messages = [
|
||||
messages: list[ChatCompletionMessageParam] = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "The cat sat on the mat.",
|
||||
@@ -382,8 +386,8 @@ async def test_chat_request(
|
||||
assert chat_embeddings.created <= completion_embeddings.created
|
||||
# Use tolerance-based comparison for embeddings
|
||||
check_embeddings_close(
|
||||
embeddings_0_lst=[d.embedding for d in chat_embeddings.data],
|
||||
embeddings_1_lst=[d.embedding for d in completion_embeddings.data],
|
||||
embeddings_0_lst=float_embeddings(chat_embeddings),
|
||||
embeddings_1_lst=float_embeddings(completion_embeddings),
|
||||
name_0="chat",
|
||||
name_1="completion",
|
||||
)
|
||||
@@ -488,7 +492,7 @@ async def test_invocations_completion_request(
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invocations_chat_request(server: RemoteOpenAIServer):
|
||||
messages = [
|
||||
messages: list[ChatCompletionMessageParam] = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "The cat sat on the mat.",
|
||||
|
||||
@@ -6,7 +6,10 @@ import openai
|
||||
import pytest
|
||||
|
||||
from tests.conftest import HfRunner
|
||||
from tests.models.language.pooling.embed_utils import run_embedding_correctness_test
|
||||
from tests.models.language.pooling.embed_utils import (
|
||||
float_embeddings,
|
||||
run_embedding_correctness_test,
|
||||
)
|
||||
from tests.models.utils import EmbedModelInfo
|
||||
from tests.utils import ROCM_EXTRA_ARGS, RemoteOpenAIServer
|
||||
from vllm.entrypoints.pooling.embed.protocol import EmbeddingResponse
|
||||
@@ -95,7 +98,7 @@ async def test_matryoshka(
|
||||
if dimensions is not None:
|
||||
assert len(embeddings.data[0].embedding) == dimensions
|
||||
|
||||
vllm_outputs = [d.embedding for d in embeddings.data]
|
||||
vllm_outputs = float_embeddings(embeddings)
|
||||
run_embedding_correctness_test(hf_model, prompts, vllm_outputs, dimensions)
|
||||
|
||||
if model_info.is_matryoshka:
|
||||
|
||||
@@ -274,6 +274,7 @@ async def test_long_text_embedding_1500_chars(
|
||||
|
||||
# Verify embedding vector validity
|
||||
embedding_vector = embeddings.data[0].embedding
|
||||
assert isinstance(embedding_vector, list)
|
||||
assert all(isinstance(x, float) for x in embedding_vector), (
|
||||
"Embedding vector should contain floats"
|
||||
)
|
||||
@@ -322,6 +323,7 @@ async def test_long_text_embedding_2500_chars(
|
||||
|
||||
# Verify embedding vector validity
|
||||
embedding_vector = embeddings.data[0].embedding
|
||||
assert isinstance(embedding_vector, list)
|
||||
assert all(isinstance(x, float) for x in embedding_vector), (
|
||||
"Embedding vector should contain floats"
|
||||
)
|
||||
@@ -409,6 +411,7 @@ async def test_chunked_vs_normal_consistency(
|
||||
|
||||
# 验证embedding向量的有效性
|
||||
embedding_vector = embeddings.data[0].embedding
|
||||
assert isinstance(embedding_vector, list)
|
||||
assert all(isinstance(x, float) for x in embedding_vector)
|
||||
assert not all(x == 0 for x in embedding_vector)
|
||||
|
||||
@@ -438,6 +441,7 @@ async def test_chunked_processing_response_format(
|
||||
|
||||
# Verify embedding vector properties
|
||||
embedding_vector = embeddings.data[0].embedding
|
||||
assert isinstance(embedding_vector, list)
|
||||
import math
|
||||
|
||||
vector_norm = math.sqrt(sum(x * x for x in embedding_vector))
|
||||
|
||||
@@ -11,6 +11,7 @@ import torch
|
||||
|
||||
from tests.models.utils import check_embeddings_close
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
|
||||
from vllm.entrypoints.pooling.pooling.protocol import PoolingResponse
|
||||
from vllm.entrypoints.pooling.utils import (
|
||||
MetadataItem,
|
||||
@@ -137,7 +138,7 @@ def test_completion_request_batched(server: RemoteOpenAIServer, model_name: str)
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
async def test_chat_request(server: RemoteOpenAIServer, model_name: str):
|
||||
messages = [
|
||||
messages: list[ChatCompletionMessageParam] = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "The cat sat on the mat.",
|
||||
@@ -524,7 +525,7 @@ async def test_invocations_chat_request(server: RemoteOpenAIServer):
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invocations_conversation_chat_request(server: RemoteOpenAIServer):
|
||||
messages = [
|
||||
messages: list[ChatCompletionMessageParam] = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "The cat sat on the mat.",
|
||||
|
||||
@@ -95,12 +95,16 @@ def test_token_type_ids_stay_aligned_with_a_truncated_padded_prompt():
|
||||
truncation_side="left",
|
||||
)
|
||||
|
||||
prompt = tok_params.apply_post_tokenization(tokenizer, prompt)
|
||||
processed_prompt = tok_params.apply_post_tokenization(
|
||||
tokenizer,
|
||||
prompt,
|
||||
)
|
||||
assert "prompt_token_ids" in processed_prompt
|
||||
token_type_ids = _apply_post_tokenization_to_token_type_ids(
|
||||
tokenizer, tok_params, token_type_ids
|
||||
)
|
||||
|
||||
prompt_token_ids = prompt["prompt_token_ids"]
|
||||
prompt_token_ids = processed_prompt["prompt_token_ids"]
|
||||
assert len(token_type_ids) == len(prompt_token_ids)
|
||||
|
||||
# Keeping the last 40 tokens drops the first 10 query tokens, so 10 query
|
||||
|
||||
@@ -30,12 +30,18 @@ def test_online_forwards_truncate_prompt_tokens_to_proxy(monkeypatch):
|
||||
silently disables truncate_prompt_tokens for Jina rerank/score.
|
||||
"""
|
||||
proc = JinaRankingIOProcessor.__new__(JinaRankingIOProcessor)
|
||||
proc.valid_inputs_online = MagicMock(
|
||||
return_value=ScoringData(data_1=["query"], data_2=["doc"])
|
||||
monkeypatch.setattr(
|
||||
proc,
|
||||
"valid_inputs_online",
|
||||
MagicMock(return_value=ScoringData(data_1=["query"], data_2=["doc"])),
|
||||
)
|
||||
monkeypatch.setattr(proc, "_get_token_limits", MagicMock(return_value=(0, 0)))
|
||||
monkeypatch.setattr(
|
||||
proc, "ensure_str", MagicMock(side_effect=lambda data: list(data))
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
proc, "format_docs_prompts_func", MagicMock(return_value="formatted prompt")
|
||||
)
|
||||
proc._get_token_limits = MagicMock(return_value=(0, 0))
|
||||
proc.ensure_str = MagicMock(side_effect=lambda data: list(data))
|
||||
proc.format_docs_prompts_func = MagicMock(return_value="formatted prompt")
|
||||
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ from unittest.mock import AsyncMock, Mock
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.pooling.scoring.serving import ServingScores
|
||||
from vllm.entrypoints.pooling.typing import PoolingServeContext
|
||||
from vllm.entrypoints.pooling.typing import PoolingEngineInput, PoolingServeContext
|
||||
from vllm.inputs import tokens_input
|
||||
from vllm.pooling_params import PoolingParams
|
||||
|
||||
@@ -21,8 +21,18 @@ def _make_context() -> PoolingServeContext:
|
||||
priorities=None,
|
||||
prompt_extras=None,
|
||||
engine_inputs=[
|
||||
tokens_input(prompt_token_ids=[1]),
|
||||
tokens_input(prompt_token_ids=[2]),
|
||||
PoolingEngineInput(
|
||||
prompts=tokens_input(prompt_token_ids=[1]),
|
||||
params=PoolingParams(task="token_embed"),
|
||||
lora_requests=None,
|
||||
priorities=0,
|
||||
),
|
||||
PoolingEngineInput(
|
||||
prompts=tokens_input(prompt_token_ids=[2]),
|
||||
params=PoolingParams(task="token_embed"),
|
||||
lora_requests=None,
|
||||
priorities=0,
|
||||
),
|
||||
],
|
||||
n_queries=1,
|
||||
)
|
||||
@@ -39,10 +49,13 @@ def _query_key(context: PoolingServeContext) -> str:
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_colliding_request_ids_use_distinct_query_cache_keys():
|
||||
async def test_colliding_request_ids_use_distinct_query_cache_keys(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
serving = object.__new__(ServingScores)
|
||||
serving._prepare_generators = AsyncMock()
|
||||
serving._collect_batch = AsyncMock()
|
||||
prepare_generators = AsyncMock()
|
||||
monkeypatch.setattr(serving, "_prepare_generators", prepare_generators)
|
||||
monkeypatch.setattr(serving, "_collect_batch", AsyncMock())
|
||||
|
||||
first = _make_context()
|
||||
second = _make_context()
|
||||
@@ -50,9 +63,7 @@ async def test_colliding_request_ids_use_distinct_query_cache_keys():
|
||||
await serving._flash_late_interaction_encode_queries(context)
|
||||
await serving._flash_late_interaction_encode_docs(context)
|
||||
|
||||
prepared_contexts = [
|
||||
call.args[0] for call in serving._prepare_generators.await_args_list
|
||||
]
|
||||
prepared_contexts = [call.args[0] for call in prepare_generators.await_args_list]
|
||||
first_query_key = _query_key(prepared_contexts[0])
|
||||
first_doc_key = _query_key(prepared_contexts[1])
|
||||
second_query_key = _query_key(prepared_contexts[2])
|
||||
|
||||
@@ -45,6 +45,7 @@ async def test_pooling_token_classify(server: RemoteOpenAIServer, model_name: st
|
||||
poolings = PoolingResponse.model_validate(response.json())
|
||||
assert len(poolings.data) == 1
|
||||
assert len(poolings.data[0].data) == 8
|
||||
assert isinstance(poolings.data[0].data[0], list)
|
||||
assert len(poolings.data[0].data[0]) == 2
|
||||
|
||||
|
||||
|
||||
@@ -71,6 +71,7 @@ async def test_pooling_token_embed(server: RemoteOpenAIServer, model_name: str):
|
||||
assert poolings.id == f"pooling-{request_id}"
|
||||
assert len(poolings.data) == 1
|
||||
assert len(poolings.data[0].data) == len(input_tokens)
|
||||
assert isinstance(poolings.data[0].data[0], list)
|
||||
assert len(poolings.data[0].data[0]) == 384
|
||||
|
||||
|
||||
|
||||
@@ -27,6 +27,11 @@ async def client(server):
|
||||
yield http_client
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def tokenizer():
|
||||
return get_tokenizer(MODEL_NAME)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -372,6 +377,58 @@ async def test_derender_chat_model_omitted_resolves_served_name(client):
|
||||
assert response.json()["model"] == MODEL_NAME
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_derender_chat_leading_space_seeded_from_prompt(client, tokenizer):
|
||||
"""prompt_token_ids keeps the first token's leading space on this
|
||||
Metaspace tokenizer, matching the coupled path."""
|
||||
gen_req = await _render_chat(client)
|
||||
prompt_token_ids = gen_req["token_ids"]
|
||||
output_ids = tokenizer.encode("Hello there, output", add_special_tokens=False)
|
||||
|
||||
unseeded = await client.post(
|
||||
"/v1/chat/completions/derender",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"generate_response": _make_generate_response(output_ids),
|
||||
},
|
||||
)
|
||||
seeded = await client.post(
|
||||
"/v1/chat/completions/derender",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"generate_response": _make_generate_response(output_ids),
|
||||
"prompt_token_ids": prompt_token_ids,
|
||||
},
|
||||
)
|
||||
assert unseeded.status_code == 200
|
||||
assert seeded.status_code == 200
|
||||
unseeded_content = unseeded.json()["choices"][0]["message"]["content"]
|
||||
seeded_content = seeded.json()["choices"][0]["message"]["content"]
|
||||
|
||||
assert not unseeded_content.startswith(" ")
|
||||
assert seeded_content.startswith(" ")
|
||||
assert seeded_content.lstrip(" ") == unseeded_content
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_derender_chat_oversized_prompt_token_ids_rejected(client):
|
||||
"""prompt_token_ids longer than max_model_len returns 400."""
|
||||
gen_req = await _render_chat(client)
|
||||
synthetic_ids = gen_req["token_ids"][:3]
|
||||
oversized_prompt_ids = [42] * 1_000_000
|
||||
|
||||
response = await client.post(
|
||||
"/v1/chat/completions/derender",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"generate_response": _make_generate_response(synthetic_ids),
|
||||
"prompt_token_ids": oversized_prompt_ids,
|
||||
},
|
||||
)
|
||||
assert response.status_code == 400
|
||||
assert "max_model_len" in response.json()["error"]["message"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Completion derender tests
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -486,6 +543,84 @@ async def test_derender_completion_prompt_tokens_length_mismatch(client):
|
||||
assert response.status_code == 400
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_derender_completion_prompt_token_ids_length_mismatch(client):
|
||||
"""len(prompt_token_ids) != len(generate_responses) returns 400, the
|
||||
same as the existing prompt_tokens length check."""
|
||||
gr1 = await _render_completion(client, "Hello")
|
||||
ids1 = gr1["token_ids"][:3]
|
||||
|
||||
response = await client.post(
|
||||
"/v1/completions/derender",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"generate_responses": [
|
||||
_make_completion_generate_response(ids1, gr1["request_id"]),
|
||||
],
|
||||
"prompt_token_ids": [[1, 2], [3, 4]],
|
||||
},
|
||||
)
|
||||
assert response.status_code == 400
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_derender_completion_leading_space_seeded_from_prompt(client, tokenizer):
|
||||
"""Completions counterpart of
|
||||
test_derender_chat_leading_space_seeded_from_prompt."""
|
||||
gr1 = await _render_completion(client, "Hello world")
|
||||
prompt_token_ids = gr1["token_ids"]
|
||||
output_ids = tokenizer.encode("Hello there, output", add_special_tokens=False)
|
||||
|
||||
unseeded = await client.post(
|
||||
"/v1/completions/derender",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"generate_responses": [
|
||||
_make_completion_generate_response(output_ids, gr1["request_id"]),
|
||||
],
|
||||
},
|
||||
)
|
||||
seeded = await client.post(
|
||||
"/v1/completions/derender",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"generate_responses": [
|
||||
_make_completion_generate_response(output_ids, gr1["request_id"]),
|
||||
],
|
||||
"prompt_token_ids": [prompt_token_ids],
|
||||
},
|
||||
)
|
||||
assert unseeded.status_code == 200
|
||||
assert seeded.status_code == 200
|
||||
unseeded_text = unseeded.json()["choices"][0]["text"]
|
||||
seeded_text = seeded.json()["choices"][0]["text"]
|
||||
|
||||
assert not unseeded_text.startswith(" ")
|
||||
assert seeded_text.startswith(" ")
|
||||
assert seeded_text.lstrip(" ") == unseeded_text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_derender_completion_oversized_prompt_token_ids_rejected(client):
|
||||
"""A prompt_token_ids entry longer than max_model_len returns 400."""
|
||||
gr1 = await _render_completion(client, "Hello")
|
||||
ids1 = gr1["token_ids"][:3]
|
||||
oversized_prompt_ids = [42] * 1_000_000
|
||||
|
||||
response = await client.post(
|
||||
"/v1/completions/derender",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"generate_responses": [
|
||||
_make_completion_generate_response(ids1, gr1["request_id"]),
|
||||
],
|
||||
"prompt_token_ids": [oversized_prompt_ids],
|
||||
},
|
||||
)
|
||||
assert response.status_code == 400
|
||||
assert "max_model_len" in response.json()["error"]["message"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_derender_completion_empty_generate_responses(client):
|
||||
"""Empty generate_responses list returns 400."""
|
||||
|
||||
@@ -20,11 +20,17 @@ Tests are split into two layers:
|
||||
import json
|
||||
import threading
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from pydantic import ValidationError
|
||||
from tokenizers.decoders import DecodeStream
|
||||
|
||||
import vllm.renderers.online_derenderer as online_derenderer_module
|
||||
from tests.entrypoints.scale_out.derender.utils import stream_chat_derender
|
||||
from tests.utils import RemoteLaunchRenderServer
|
||||
from vllm.entrypoints.generate.base.protocol import (
|
||||
DeltaFunctionCall,
|
||||
DeltaMessage,
|
||||
@@ -32,7 +38,17 @@ from vllm.entrypoints.generate.base.protocol import (
|
||||
PerRequestMetrics,
|
||||
SpeculativeDecodingMetrics,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
ChatCompletionStreamResponse,
|
||||
)
|
||||
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
||||
from vllm.entrypoints.scale_out.derender.serving import ServingDerender
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
|
||||
DerenderChatRequest,
|
||||
DerenderChatStreamRequest,
|
||||
DerenderCompletionRequest,
|
||||
DerenderCompletionStreamRequest,
|
||||
DerenderStreamState,
|
||||
GenerateResponse,
|
||||
GenerateResponseChoice,
|
||||
@@ -41,6 +57,12 @@ from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
|
||||
)
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse, UsageInfo
|
||||
from vllm.parser import Parser
|
||||
from vllm.renderers.online_derenderer import (
|
||||
OnlineDerenderer,
|
||||
_decode_params,
|
||||
_seed_stream_state,
|
||||
)
|
||||
from vllm.tokenizers import get_tokenizer
|
||||
from vllm.utils import random_uuid
|
||||
|
||||
MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
|
||||
@@ -169,6 +191,8 @@ def _make_stream_chunk(
|
||||
request_id: str = "test-req",
|
||||
usage: dict | None = None,
|
||||
metrics: PerRequestMetrics | None = None,
|
||||
logprobs: dict | None = None,
|
||||
prompt_token_ids: list[int] | None = None,
|
||||
) -> GenerateStreamResponse:
|
||||
"""Build a GenerateStreamResponse SSE chunk."""
|
||||
return GenerateStreamResponse(
|
||||
@@ -178,13 +202,31 @@ def _make_stream_chunk(
|
||||
index=index,
|
||||
token_ids=token_ids,
|
||||
finish_reason=finish_reason,
|
||||
logprobs=logprobs,
|
||||
)
|
||||
],
|
||||
usage=UsageInfo(**usage) if usage else None,
|
||||
metrics=metrics,
|
||||
prompt_token_ids=prompt_token_ids,
|
||||
)
|
||||
|
||||
|
||||
def _placeholder_logprobs(token_ids: list[int]) -> dict:
|
||||
"""Per-token logprob entries using token_id:N placeholders, as sent by
|
||||
the generate worker."""
|
||||
return {
|
||||
"content": [
|
||||
{
|
||||
"token": f"token_id:{tid}",
|
||||
"logprob": -0.5,
|
||||
"bytes": None,
|
||||
"top_logprobs": [],
|
||||
}
|
||||
for tid in token_ids
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
def _make_usage_chunk(
|
||||
completion_tokens: int,
|
||||
prompt_tokens: int = 0,
|
||||
@@ -210,18 +252,12 @@ def _make_usage_chunk(
|
||||
@pytest.fixture(scope="module")
|
||||
def tokenizer():
|
||||
"""Load the tiny tokenizer used across unit tests."""
|
||||
from vllm.tokenizers import get_tokenizer
|
||||
|
||||
return get_tokenizer(MODEL_NAME)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def derenderer(tokenizer, request):
|
||||
"""Construct a minimal OnlineDerenderer backed by a stub renderer."""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from vllm.renderers.online_derenderer import OnlineDerenderer
|
||||
|
||||
renderer = MagicMock()
|
||||
renderer.get_tokenizer.return_value = tokenizer
|
||||
|
||||
@@ -255,10 +291,6 @@ def parsed_derenderer(tokenizer):
|
||||
`renderer._executor` must be a real `ThreadPoolExecutor`.
|
||||
`loop.run_in_executor` cannot submit work to a `MagicMock`.
|
||||
"""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from vllm.renderers.online_derenderer import OnlineDerenderer
|
||||
|
||||
renderer = MagicMock()
|
||||
renderer.get_tokenizer.return_value = tokenizer
|
||||
renderer._executor = ThreadPoolExecutor(max_workers=2)
|
||||
@@ -278,6 +310,45 @@ def parsed_derenderer(tokenizer):
|
||||
return dr
|
||||
|
||||
|
||||
def test_non_streaming_derender_initializes_parser_prefix(
|
||||
derenderer, tokenizer, monkeypatch
|
||||
):
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from vllm.entrypoints.generate.base.protocol import FunctionCall
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
|
||||
parser = MagicMock()
|
||||
parser.parse.return_value = (
|
||||
None,
|
||||
None,
|
||||
[FunctionCall(name="tool", arguments="{}")],
|
||||
)
|
||||
monkeypatch.setattr(derenderer, "parser", MagicMock(return_value=parser))
|
||||
generated_ids = tokenizer.encode("answer", add_special_tokens=False)
|
||||
response = GenerateResponse(
|
||||
request_id="test",
|
||||
prompt_token_ids=[11, 12],
|
||||
choices=[
|
||||
GenerateResponseChoice(
|
||||
index=0,
|
||||
token_ids=generated_ids,
|
||||
finish_reason="length",
|
||||
)
|
||||
],
|
||||
)
|
||||
request = ChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[{"role": "user", "content": "question"}],
|
||||
)
|
||||
|
||||
choices = derenderer._derender_chat(response, request)
|
||||
|
||||
parser.set_prompt_token_ids.assert_called_once_with([11, 12])
|
||||
assert choices[0].message.tool_calls[0].function.name == "tool"
|
||||
assert choices[0].finish_reason == "length"
|
||||
|
||||
|
||||
class TestDetokenizeDelta:
|
||||
"""_detokenize_delta: chunked decode must equal one shot decode."""
|
||||
|
||||
@@ -376,6 +447,57 @@ class TestDetokenizeDelta:
|
||||
# Bounded by a small constant independent of len(token_ids)
|
||||
assert max_window <= 32
|
||||
|
||||
def test_truncated_multibyte_char_is_held_back(self, derenderer, tokenizer):
|
||||
"""Output ending mid-character emits no U+FFFD: batch == stream == engine."""
|
||||
emoji_ids = tokenizer.encode("🌈", add_special_tokens=False)
|
||||
byte_ids = emoji_ids[-4:]
|
||||
assert len(byte_ids) == 4
|
||||
output_ids = byte_ids[:-1]
|
||||
|
||||
batch, _ = derenderer._detokenize_delta(
|
||||
tokenizer, output_ids, DerenderStreamState()
|
||||
)
|
||||
streamed = self._chunked(derenderer, tokenizer, [[t] for t in output_ids])
|
||||
engine = _decode_stream_expected(tokenizer, [], output_ids)
|
||||
|
||||
assert "�" not in batch
|
||||
assert batch == streamed == engine
|
||||
|
||||
full, _ = derenderer._detokenize_delta(
|
||||
tokenizer, byte_ids, DerenderStreamState()
|
||||
)
|
||||
assert full.endswith("🌈")
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("skip", "spaces", "preserve", "fast", "expected"),
|
||||
[
|
||||
(True, True, False, True, (True, True)),
|
||||
(True, False, False, True, (True, True)),
|
||||
(True, False, False, False, (True, False)),
|
||||
(False, False, False, True, (False, False)),
|
||||
(False, False, True, True, (False, False)),
|
||||
(True, False, True, True, (False, False)),
|
||||
(True, True, True, True, (False, True)),
|
||||
],
|
||||
)
|
||||
def test_decode_params_mirror_engine(
|
||||
self, monkeypatch, skip, spaces, preserve, fast, expected
|
||||
):
|
||||
"""Flags match the engine's fast or slow detokenizer for the tokenizer."""
|
||||
monkeypatch.setattr(
|
||||
online_derenderer_module, "uses_fast_detokenizer", lambda _: fast
|
||||
)
|
||||
request = CompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
prompt="x",
|
||||
skip_special_tokens=skip,
|
||||
spaces_between_special_tokens=spaces,
|
||||
)
|
||||
assert _decode_params(None, request, preserve_special=preserve) == expected
|
||||
|
||||
def test_decode_params_default_without_request(self, tokenizer):
|
||||
assert _decode_params(tokenizer, None) == (True, True)
|
||||
|
||||
def test_n_independent_streams_same_result(self, derenderer, tokenizer):
|
||||
"""N parallel streams with the same token sequence give the same text."""
|
||||
token_ids = tokenizer.encode("parallel streams")[:8]
|
||||
@@ -394,6 +516,242 @@ class TestDetokenizeDelta:
|
||||
assert results[0] == self._one_shot(tokenizer, token_ids)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Prompt seeded detokenization (matches the engine's primed detokenizer)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _decode_stream_expected(
|
||||
tokenizer, prompt_ids: list[int], output_ids: list[int]
|
||||
) -> str:
|
||||
"""Engine-equivalent decode via `DecodeStream`, primed by stepping the
|
||||
prompt (`DecodeStream(ids=...)` needs tokenizers >= 0.22)."""
|
||||
stream = DecodeStream(skip_special_tokens=True)
|
||||
for tid in prompt_ids:
|
||||
stream.step(tokenizer.backend_tokenizer, tid)
|
||||
return "".join(
|
||||
s
|
||||
for tid in output_ids
|
||||
if (s := stream.step(tokenizer.backend_tokenizer, tid)) is not None
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def leading_space_ids(tokenizer):
|
||||
"""(prompt_ids, output_ids, expected) where seeded and unseeded decode
|
||||
differ by the leading space."""
|
||||
prompt_ids = tokenizer.encode("[INST] Hi [/INST]", add_special_tokens=False)
|
||||
output_ids = tokenizer.encode("Hello there, output", add_special_tokens=False)
|
||||
expected = _decode_stream_expected(tokenizer, prompt_ids, output_ids)
|
||||
unseeded = tokenizer.decode(output_ids, skip_special_tokens=True)
|
||||
assert expected.startswith(" ") and not unseeded.startswith(" "), (
|
||||
"fixture no longer reproduces the leading-space divergence; "
|
||||
"pick a different prompt/output pair"
|
||||
)
|
||||
return prompt_ids, output_ids, expected
|
||||
|
||||
|
||||
class TestSeedStreamState:
|
||||
"""`_seed_stream_state`: pure helper building the initial decode state."""
|
||||
|
||||
def test_seeds_prev_tokens_from_prompt_tail(self, tokenizer, leading_space_ids):
|
||||
prompt_ids, _, _ = leading_space_ids
|
||||
state = _seed_stream_state(tokenizer, prompt_ids, skip_special_tokens=True)
|
||||
assert state.prev_tokens
|
||||
assert state.read_offset == len(state.prev_tokens)
|
||||
|
||||
def test_none_prompt_token_ids_returns_empty_state(self, tokenizer):
|
||||
assert (
|
||||
_seed_stream_state(tokenizer, None, skip_special_tokens=True)
|
||||
== DerenderStreamState()
|
||||
)
|
||||
|
||||
def test_empty_prompt_token_ids_returns_empty_state(self, tokenizer):
|
||||
assert (
|
||||
_seed_stream_state(tokenizer, [], skip_special_tokens=True)
|
||||
== DerenderStreamState()
|
||||
)
|
||||
|
||||
|
||||
class TestPromptSeededLeadingSpace:
|
||||
"""Every decode entry point keeps the first token's leading space when
|
||||
given prompt_token_ids."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_batch_seeded_via_explicit_prompt_token_ids(
|
||||
self, derenderer, leading_space_ids
|
||||
):
|
||||
prompt_ids, output_ids, expected = leading_space_ids
|
||||
choices = await derenderer.derender_chat(
|
||||
GenerateResponse(
|
||||
request_id="t",
|
||||
choices=[
|
||||
GenerateResponseChoice(
|
||||
index=0, token_ids=output_ids, finish_reason="stop"
|
||||
)
|
||||
],
|
||||
),
|
||||
prompt_token_ids=prompt_ids,
|
||||
)
|
||||
assert choices[0].message.content == expected
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_batch_seeded_via_generate_response_fallback(
|
||||
self, derenderer, leading_space_ids
|
||||
):
|
||||
prompt_ids, output_ids, expected = leading_space_ids
|
||||
choices = await derenderer.derender_chat(
|
||||
GenerateResponse(
|
||||
request_id="t",
|
||||
choices=[
|
||||
GenerateResponseChoice(
|
||||
index=0, token_ids=output_ids, finish_reason="stop"
|
||||
)
|
||||
],
|
||||
prompt_token_ids=prompt_ids,
|
||||
),
|
||||
)
|
||||
assert choices[0].message.content == expected
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_batch_explicit_takes_precedence_over_fallback(
|
||||
self, derenderer, tokenizer, leading_space_ids
|
||||
):
|
||||
prompt_ids, output_ids, expected = leading_space_ids
|
||||
wrong_ids = tokenizer.encode("a different prompt", add_special_tokens=False)
|
||||
choices = await derenderer.derender_chat(
|
||||
GenerateResponse(
|
||||
request_id="t",
|
||||
choices=[
|
||||
GenerateResponseChoice(
|
||||
index=0, token_ids=output_ids, finish_reason="stop"
|
||||
)
|
||||
],
|
||||
prompt_token_ids=wrong_ids,
|
||||
),
|
||||
prompt_token_ids=prompt_ids,
|
||||
)
|
||||
assert choices[0].message.content == expected
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_batch_without_prompt_context_stays_unseeded(
|
||||
self, derenderer, leading_space_ids
|
||||
):
|
||||
_, output_ids, _ = leading_space_ids
|
||||
choices = await derenderer.derender_chat(
|
||||
GenerateResponse(
|
||||
request_id="t",
|
||||
choices=[
|
||||
GenerateResponseChoice(
|
||||
index=0, token_ids=output_ids, finish_reason="stop"
|
||||
)
|
||||
],
|
||||
),
|
||||
)
|
||||
assert not choices[0].message.content.startswith(" ")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_completion_batch_seeded(self, derenderer, leading_space_ids):
|
||||
prompt_ids, output_ids, expected = leading_space_ids
|
||||
choices, _, _ = await derenderer.derender_completion(
|
||||
[
|
||||
GenerateResponse(
|
||||
request_id="t",
|
||||
choices=[
|
||||
GenerateResponseChoice(
|
||||
index=0, token_ids=output_ids, finish_reason="stop"
|
||||
)
|
||||
],
|
||||
)
|
||||
],
|
||||
prompt_token_ids=[prompt_ids],
|
||||
)
|
||||
assert choices[0].text == expected
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("from_generate_chunk", [False, True])
|
||||
async def test_chat_stream_first_chunk_seeded(
|
||||
self, derenderer, leading_space_ids, from_generate_chunk
|
||||
):
|
||||
prompt_ids, output_ids, expected = leading_space_ids
|
||||
mid = len(output_ids) // 2
|
||||
|
||||
chunk1, state = await derenderer.derender_chat_stream(
|
||||
model=MODEL_NAME,
|
||||
generate_chunk=_make_stream_chunk(
|
||||
output_ids[:mid],
|
||||
prompt_token_ids=prompt_ids if from_generate_chunk else None,
|
||||
),
|
||||
prompt_token_ids=None if from_generate_chunk else prompt_ids,
|
||||
)
|
||||
# Only needed on the first chunk; the carried state already holds
|
||||
# the seeded window from here on.
|
||||
chunk2, _ = await derenderer.derender_chat_stream(
|
||||
model=MODEL_NAME,
|
||||
generate_chunk=_make_stream_chunk(output_ids[mid:], finish_reason="stop"),
|
||||
state=state,
|
||||
)
|
||||
|
||||
streamed = (chunk1.choices[0].delta.content or "") + (
|
||||
chunk2.choices[0].delta.content or ""
|
||||
)
|
||||
assert streamed == expected
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("from_generate_chunk", [False, True])
|
||||
async def test_completion_stream_first_chunk_seeded(
|
||||
self, derenderer, leading_space_ids, from_generate_chunk
|
||||
):
|
||||
prompt_ids, output_ids, expected = leading_space_ids
|
||||
mid = len(output_ids) // 2
|
||||
|
||||
chunk1, state = await derenderer.derender_completion_stream(
|
||||
model=MODEL_NAME,
|
||||
generate_chunk=_make_stream_chunk(
|
||||
output_ids[:mid],
|
||||
prompt_token_ids=prompt_ids if from_generate_chunk else None,
|
||||
),
|
||||
prompt_token_ids=None if from_generate_chunk else prompt_ids,
|
||||
)
|
||||
chunk2, _ = await derenderer.derender_completion_stream(
|
||||
model=MODEL_NAME,
|
||||
generate_chunk=_make_stream_chunk(output_ids[mid:], finish_reason="stop"),
|
||||
state=state,
|
||||
)
|
||||
|
||||
assert chunk1.choices[0].text + chunk2.choices[0].text == expected
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_parser_path_ephemeral_state_seeded(
|
||||
self, parsed_derenderer, monkeypatch, leading_space_ids
|
||||
):
|
||||
"""The parser path's per-call detok window is seeded too, so
|
||||
`parse_delta` sees the same text as coupled serving."""
|
||||
|
||||
class _EchoParser(Parser):
|
||||
def parse_delta(
|
||||
self,
|
||||
delta_text,
|
||||
delta_token_ids,
|
||||
request,
|
||||
prompt_token_ids=None,
|
||||
*,
|
||||
finished,
|
||||
):
|
||||
return DeltaMessage(content=delta_text) if delta_text else None
|
||||
|
||||
monkeypatch.setattr(parsed_derenderer, "parser", _EchoParser)
|
||||
prompt_ids, output_ids, expected = leading_space_ids
|
||||
|
||||
chunk, _ = await parsed_derenderer.derender_chat_stream(
|
||||
model=MODEL_NAME,
|
||||
generate_chunk=_make_stream_chunk(output_ids, finish_reason="stop"),
|
||||
chat_request=_chat_request(),
|
||||
prompt_token_ids=prompt_ids,
|
||||
)
|
||||
assert chunk.choices[0].delta.content == expected
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"stream_method",
|
||||
@@ -403,8 +761,6 @@ async def test_streaming_detokenization_runs_off_event_loop(
|
||||
derenderer, monkeypatch, stream_method
|
||||
):
|
||||
"""Streaming detokenization runs on the renderer executor."""
|
||||
import vllm.renderers.online_derenderer as online_derenderer_module
|
||||
|
||||
event_loop_thread_id = threading.get_ident()
|
||||
executor_thread_id = derenderer.renderer._executor.submit(
|
||||
threading.get_ident
|
||||
@@ -504,8 +860,6 @@ class TestDerenderCompletionStream:
|
||||
@pytest.mark.asyncio
|
||||
async def test_skip_special_tokens_threaded(self, derenderer, tokenizer):
|
||||
"""completion_request.skip_special_tokens is honored (not hardcoded True)."""
|
||||
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
||||
|
||||
eos = tokenizer.eos_token_id
|
||||
if eos is None:
|
||||
pytest.skip("tokenizer has no eos token to exercise special stripping")
|
||||
@@ -536,6 +890,115 @@ class TestDerenderCompletionStream:
|
||||
assert chunk.choices[0].finish_reason == "length"
|
||||
|
||||
|
||||
class TestStreamLogprobs:
|
||||
"""Streaming derender must carry per-chunk logprobs with placeholders
|
||||
resolved, matching what the generate streaming path emits."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_stream_logprobs_resolved_per_chunk(self, derenderer, tokenizer):
|
||||
"""Each streamed chunk carries logprobs with token_id:N resolved."""
|
||||
token_ids = tokenizer.encode("hello world")[:6]
|
||||
mid = len(token_ids) // 2
|
||||
|
||||
state = None
|
||||
for part in (token_ids[:mid], token_ids[mid:]):
|
||||
chunk, state = await derenderer.derender_chat_stream(
|
||||
model=MODEL_NAME,
|
||||
generate_chunk=_make_stream_chunk(
|
||||
part, logprobs=_placeholder_logprobs(part)
|
||||
),
|
||||
state=state,
|
||||
)
|
||||
logprobs = chunk.choices[0].logprobs
|
||||
assert logprobs is not None and logprobs.content is not None
|
||||
assert len(logprobs.content) == len(part)
|
||||
for entry in logprobs.content:
|
||||
assert not entry.token.startswith("token_id:"), (
|
||||
f"placeholder not resolved: {entry.token!r}"
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_stream_logprobs_multibyte_across_chunks(
|
||||
self, derenderer, tokenizer
|
||||
):
|
||||
"""Byte-fallback correction works when a multi-byte character's
|
||||
tokens are split across chunks (context carried in stream_state)."""
|
||||
token_ids = tokenizer.encode("👍", add_special_tokens=False)
|
||||
if len(token_ids) < 2 or "�" not in tokenizer.decode([token_ids[-1]]):
|
||||
pytest.skip("Tokenizer does not byte-split this character")
|
||||
|
||||
state = None
|
||||
last_chunk = None
|
||||
# One token per chunk: every entry after the first needs cross-chunk
|
||||
# context to resolve without U+FFFD.
|
||||
for i, tid in enumerate(token_ids):
|
||||
last_chunk, state = await derenderer.derender_chat_stream(
|
||||
model=MODEL_NAME,
|
||||
generate_chunk=_make_stream_chunk(
|
||||
[tid], logprobs=_placeholder_logprobs([tid])
|
||||
),
|
||||
state=state,
|
||||
)
|
||||
|
||||
assert last_chunk is not None
|
||||
final_entry = last_chunk.choices[0].logprobs.content[0]
|
||||
assert not final_entry.token.endswith("�"), (
|
||||
"byte-fallback correction failed across chunk boundary: "
|
||||
f"{final_entry.token!r}"
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_completion_stream_logprobs_text_offset_absolute(
|
||||
self, derenderer, tokenizer
|
||||
):
|
||||
"""text_offset continues across chunks instead of restarting at 0."""
|
||||
token_ids = tokenizer.encode("hello world")[:6]
|
||||
mid = len(token_ids) // 2
|
||||
|
||||
chunk1, state = await derenderer.derender_completion_stream(
|
||||
model=MODEL_NAME,
|
||||
generate_chunk=_make_stream_chunk(
|
||||
token_ids[:mid], logprobs=_placeholder_logprobs(token_ids[:mid])
|
||||
),
|
||||
)
|
||||
chunk2, _ = await derenderer.derender_completion_stream(
|
||||
model=MODEL_NAME,
|
||||
generate_chunk=_make_stream_chunk(
|
||||
token_ids[mid:], logprobs=_placeholder_logprobs(token_ids[mid:])
|
||||
),
|
||||
state=state,
|
||||
)
|
||||
|
||||
assert chunk1.choices[0].logprobs.text_offset[0] == 0
|
||||
assert chunk2.choices[0].logprobs.text_offset[0] == len(chunk1.choices[0].text)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_logprob_state_stays_bounded(self, derenderer, tokenizer):
|
||||
"""Carried logprob context never exceeds the 4-token window."""
|
||||
token_ids = tokenizer.encode("the quick brown fox jumps over the lazy dog")
|
||||
state = None
|
||||
for tid in token_ids:
|
||||
_, state = await derenderer.derender_chat_stream(
|
||||
model=MODEL_NAME,
|
||||
generate_chunk=_make_stream_chunk(
|
||||
[tid], logprobs=_placeholder_logprobs([tid])
|
||||
),
|
||||
state=state,
|
||||
)
|
||||
assert state is not None
|
||||
assert len(state.logprob_context_token_ids) <= 4
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_without_logprobs_unchanged(self, derenderer, tokenizer):
|
||||
"""Chunks without logprobs keep logprobs=None on the output choice."""
|
||||
token_ids = tokenizer.encode("hello")[:3]
|
||||
chunk, _ = await derenderer.derender_chat_stream(
|
||||
model=MODEL_NAME,
|
||||
generate_chunk=_make_stream_chunk(token_ids),
|
||||
)
|
||||
assert chunk.choices[0].logprobs is None
|
||||
|
||||
|
||||
class TestDerenderChatStream:
|
||||
"""derender_chat_stream: plain detok branch (no parser)."""
|
||||
|
||||
@@ -623,8 +1086,6 @@ class TestDerenderChatStream:
|
||||
|
||||
|
||||
def _chat_request(**kwargs):
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
|
||||
kwargs.setdefault("messages", [{"role": "user", "content": "hi"}])
|
||||
kwargs.setdefault("model", MODEL_NAME)
|
||||
return ChatCompletionRequest(**kwargs)
|
||||
@@ -906,8 +1367,6 @@ def harmony_encode():
|
||||
@pytest.fixture(scope="module")
|
||||
def harmony_tokenizer():
|
||||
pytest.importorskip("openai_harmony")
|
||||
from vllm.tokenizers import get_tokenizer
|
||||
|
||||
return get_tokenizer(HARMONY_MODEL, trust_remote_code=True)
|
||||
|
||||
|
||||
@@ -919,10 +1378,7 @@ def harmony_derenderer(harmony_tokenizer):
|
||||
with a real tokenizer and parser, so the replay path is exercised
|
||||
against Harmony's actual channel grammar rather than a stub.
|
||||
"""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from vllm.parser.harmony import HarmonyParser
|
||||
from vllm.renderers.online_derenderer import OnlineDerenderer
|
||||
|
||||
renderer = MagicMock()
|
||||
renderer.get_tokenizer.return_value = harmony_tokenizer
|
||||
@@ -1058,20 +1514,14 @@ class TestDerenderStreamStateValidation:
|
||||
"""DerenderStreamState rejects malformed caller supplied offsets/lengths."""
|
||||
|
||||
def test_negative_prefix_offset_rejected(self):
|
||||
from pydantic import ValidationError
|
||||
|
||||
with pytest.raises(ValidationError):
|
||||
DerenderStreamState(prefix_offset=-1)
|
||||
|
||||
def test_negative_read_offset_rejected(self):
|
||||
from pydantic import ValidationError
|
||||
|
||||
with pytest.raises(ValidationError):
|
||||
DerenderStreamState(read_offset=-1)
|
||||
|
||||
def test_prev_tokens_over_cap_rejected(self):
|
||||
from pydantic import ValidationError
|
||||
|
||||
with pytest.raises(ValidationError):
|
||||
DerenderStreamState(prev_tokens=["a"] * 1025)
|
||||
|
||||
@@ -1080,14 +1530,10 @@ class TestDerenderStreamStateValidation:
|
||||
assert len(state.prev_tokens) == 1024
|
||||
|
||||
def test_output_chunk_lens_mismatch_rejected(self):
|
||||
from pydantic import ValidationError
|
||||
|
||||
with pytest.raises(ValidationError):
|
||||
DerenderStreamState(output_token_ids=[1, 2, 3], output_chunk_lens=[1, 1])
|
||||
|
||||
def test_output_chunk_lens_zero_entry_rejected(self):
|
||||
from pydantic import ValidationError
|
||||
|
||||
with pytest.raises(ValidationError):
|
||||
DerenderStreamState(output_token_ids=[1, 2], output_chunk_lens=[0, 2])
|
||||
|
||||
@@ -1098,14 +1544,40 @@ class TestDerenderStreamStateValidation:
|
||||
assert state.output_chunk_lens == [1, 2]
|
||||
|
||||
|
||||
_GEN = {"request_id": "t", "choices": []}
|
||||
_GEN_BAD = {**_GEN, "prompt_token_ids": [-1]}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_cls,payload",
|
||||
[
|
||||
(DerenderChatRequest, {"generate_response": _GEN, "prompt_token_ids": [-1]}),
|
||||
(DerenderChatRequest, {"generate_response": _GEN_BAD}),
|
||||
(
|
||||
DerenderCompletionRequest,
|
||||
{"generate_responses": [_GEN], "prompt_token_ids": [[-1]]},
|
||||
),
|
||||
(
|
||||
DerenderChatStreamRequest,
|
||||
{"stream": True, "generate_chunk": _GEN, "prompt_token_ids": [-1]},
|
||||
),
|
||||
(
|
||||
DerenderCompletionStreamRequest,
|
||||
{"stream": True, "generate_chunk": _GEN_BAD},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_negative_prompt_token_ids_rejected(model_cls, payload):
|
||||
"""Negative prompt ids would reach convert_ids_to_tokens, so they get a
|
||||
400 at parse time like `GenerateRequest.token_ids` does."""
|
||||
with pytest.raises(ValidationError, match="prompt_token_ids"):
|
||||
model_cls.model_validate(payload)
|
||||
|
||||
|
||||
class TestServingDerenderStreamErrorHandling:
|
||||
"""Malformed stream_state must surface as 400 and not an unhandled 500."""
|
||||
|
||||
def _make_serving(self, side_effect: Exception):
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
from vllm.entrypoints.scale_out.derender.serving import ServingDerender
|
||||
|
||||
models = MagicMock()
|
||||
models.is_base_model.return_value = True
|
||||
models.model_config = MagicMock()
|
||||
@@ -1123,10 +1595,6 @@ class TestServingDerenderStreamErrorHandling:
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("exc", [KeyError("bad byte"), IndexError("oob")])
|
||||
async def test_completion_stream_bad_state_returns_400(self, exc):
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
|
||||
DerenderCompletionStreamRequest,
|
||||
)
|
||||
|
||||
serving = self._make_serving(exc)
|
||||
request = DerenderCompletionStreamRequest(
|
||||
stream=True,
|
||||
@@ -1141,10 +1609,6 @@ class TestServingDerenderStreamErrorHandling:
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("exc", [KeyError("bad byte"), IndexError("oob")])
|
||||
async def test_chat_stream_bad_state_returns_400(self, exc):
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
|
||||
DerenderChatStreamRequest,
|
||||
)
|
||||
|
||||
serving = self._make_serving(exc)
|
||||
request = DerenderChatStreamRequest(
|
||||
stream=True,
|
||||
@@ -1163,13 +1627,6 @@ class TestServingDerenderStreamValidation:
|
||||
before touching the tokenizer)."""
|
||||
|
||||
def _make_serving(self, *, parser_configured: bool, max_model_len: int = 100_000):
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionStreamResponse,
|
||||
)
|
||||
from vllm.entrypoints.scale_out.derender.serving import ServingDerender
|
||||
|
||||
models = MagicMock()
|
||||
models.is_base_model.return_value = True
|
||||
models.model_config = MagicMock()
|
||||
@@ -1187,10 +1644,6 @@ class TestServingDerenderStreamValidation:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_missing_chat_request_with_parser_rejected(self):
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
|
||||
DerenderChatStreamRequest,
|
||||
)
|
||||
|
||||
serving = self._make_serving(parser_configured=True)
|
||||
request = DerenderChatStreamRequest(
|
||||
stream=True,
|
||||
@@ -1205,10 +1658,6 @@ class TestServingDerenderStreamValidation:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_missing_chat_request_without_parser_ok(self):
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
|
||||
DerenderChatStreamRequest,
|
||||
)
|
||||
|
||||
serving = self._make_serving(parser_configured=False)
|
||||
request = DerenderChatStreamRequest(
|
||||
stream=True,
|
||||
@@ -1224,10 +1673,6 @@ class TestServingDerenderStreamValidation:
|
||||
the same way it rejects a missing chat_request. Without it,
|
||||
parse_delta cannot tell whether the prompt left reasoning open and
|
||||
would misclassify reasoning content as plain content."""
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
|
||||
DerenderChatStreamRequest,
|
||||
)
|
||||
|
||||
serving = self._make_serving(parser_configured=True)
|
||||
request = DerenderChatStreamRequest(
|
||||
stream=True,
|
||||
@@ -1243,10 +1688,6 @@ class TestServingDerenderStreamValidation:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_prompt_token_ids_present_with_parser_ok(self):
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
|
||||
DerenderChatStreamRequest,
|
||||
)
|
||||
|
||||
serving = self._make_serving(parser_configured=True)
|
||||
request = DerenderChatStreamRequest(
|
||||
stream=True,
|
||||
@@ -1260,10 +1701,6 @@ class TestServingDerenderStreamValidation:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_oversized_output_token_ids_rejected(self):
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
|
||||
DerenderChatStreamRequest,
|
||||
)
|
||||
|
||||
serving = self._make_serving(parser_configured=False, max_model_len=4)
|
||||
request = DerenderChatStreamRequest(
|
||||
stream=True,
|
||||
@@ -1285,10 +1722,6 @@ class TestServingDerenderStreamValidation:
|
||||
parser configured deployment rescans it in full on every chunk
|
||||
(is_reasoning_end / adjust_initial_state_from_prompt), so it must be
|
||||
bounded the same way output_token_ids is."""
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
|
||||
DerenderChatStreamRequest,
|
||||
)
|
||||
|
||||
serving = self._make_serving(parser_configured=True, max_model_len=4)
|
||||
request = DerenderChatStreamRequest(
|
||||
stream=True,
|
||||
@@ -1309,10 +1742,6 @@ class TestServingDerenderStreamValidation:
|
||||
DerenderStreamState is threaded through every choice). The check
|
||||
could never fire since derender_chat_stream itself rejects anything
|
||||
above 1 first."""
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
|
||||
DerenderChatStreamRequest,
|
||||
)
|
||||
|
||||
serving = self._make_serving(parser_configured=False)
|
||||
two_choices = GenerateStreamResponse(
|
||||
request_id="t",
|
||||
@@ -1337,16 +1766,12 @@ class TestServingDerenderStreamValidation:
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
from tests.utils import RemoteLaunchRenderServer
|
||||
|
||||
with RemoteLaunchRenderServer(MODEL_NAME, []) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def client(server):
|
||||
import httpx
|
||||
|
||||
async with httpx.AsyncClient(
|
||||
base_url=server.url_for(""), timeout=30.0
|
||||
) as http_client:
|
||||
@@ -1610,8 +2035,6 @@ _PARSER_TOOLS = [
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def parser_server():
|
||||
from tests.utils import RemoteLaunchRenderServer
|
||||
|
||||
args = [
|
||||
"--enable-auto-tool-choice",
|
||||
"--tool-call-parser",
|
||||
@@ -1625,8 +2048,6 @@ def parser_server():
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def parser_client(parser_server):
|
||||
import httpx
|
||||
|
||||
async with httpx.AsyncClient(
|
||||
base_url=parser_server.url_for(""), timeout=60.0
|
||||
) as http_client:
|
||||
@@ -1635,8 +2056,6 @@ async def parser_client(parser_server):
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def parser_tokenizer():
|
||||
from vllm.tokenizers import get_tokenizer
|
||||
|
||||
return get_tokenizer(PARSER_MODEL)
|
||||
|
||||
|
||||
|
||||
@@ -15,15 +15,26 @@ from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.mm_features import (
|
||||
placeholder_ranges_from_engine_input,
|
||||
)
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.mm_serde import (
|
||||
encode_mm_kwargs_item,
|
||||
)
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
|
||||
GenerateRequest,
|
||||
GenerateResponse,
|
||||
MultiModalFeatures,
|
||||
PlaceholderRangeInfo,
|
||||
)
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.serving import ServingTokens
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.exceptions import GenerationError
|
||||
from vllm.logprobs import Logprob
|
||||
from vllm.multimodal.inputs import PlaceholderRange
|
||||
from vllm.outputs import CompletionOutput, RequestOutput
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalBatchedField,
|
||||
MultiModalFieldElem,
|
||||
MultiModalKwargsItem,
|
||||
PlaceholderRange,
|
||||
)
|
||||
from vllm.outputs import CompletionOutput, RequestOutput, SamplingMask
|
||||
from vllm.renderers import renderer_from_config
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
from vllm.sampling_params import SamplingParams
|
||||
@@ -285,6 +296,46 @@ async def test_non_stream_error():
|
||||
await serving.serve_tokens(request)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_serve_tokens_rejects_invalid_shm_handle_before_engine_handoff():
|
||||
engine = _mock_engine()
|
||||
engine.generate = MagicMock()
|
||||
serving = _build_serving_tokens(engine)
|
||||
cache = MagicMock()
|
||||
cache.validate_input_item.side_effect = ValueError(
|
||||
"Invalid SHM handle signature for cache key."
|
||||
)
|
||||
serving.online_renderer.renderer._mm_processor_cache = cache
|
||||
forged_item = MultiModalKwargsItem(
|
||||
{
|
||||
"address": MultiModalFieldElem(data=224, field=MultiModalBatchedField()),
|
||||
"monotonic_id": MultiModalFieldElem(
|
||||
data=4242,
|
||||
field=MultiModalBatchedField(),
|
||||
),
|
||||
}
|
||||
)
|
||||
request = GenerateRequest(
|
||||
token_ids=[1, 2, 3],
|
||||
sampling_params=SamplingParams(max_tokens=1),
|
||||
model=MODEL_NAME,
|
||||
features=MultiModalFeatures(
|
||||
mm_hashes={"image": ["image_A"]},
|
||||
mm_placeholders={"image": [PlaceholderRangeInfo(offset=0, length=1)]},
|
||||
kwargs_data={"image": [encode_mm_kwargs_item(forged_item)]},
|
||||
),
|
||||
)
|
||||
|
||||
response = await serving.serve_tokens(request)
|
||||
|
||||
assert isinstance(response, ErrorResponse)
|
||||
cache.validate_input_item.assert_called_once()
|
||||
validated_item, validated_hash = cache.validate_input_item.call_args.args
|
||||
assert validated_hash == "image_A"
|
||||
assert validated_item["address"].data == 224
|
||||
engine.generate.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_basic():
|
||||
"""Streaming returns SSE chunks with correct token_ids and ends with [DONE]."""
|
||||
@@ -1103,3 +1154,44 @@ async def test_stream_prompt_tokens_details_zero_cached():
|
||||
# Zero cached tokens must be present, not omitted
|
||||
assert usage_chunk["usage"]["prompt_tokens_details"] is not None
|
||||
assert usage_chunk["usage"]["prompt_tokens_details"]["cached_tokens"] == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("with_mask", [False, True])
|
||||
async def test_stream_sampling_mask_matches_each_token_chunk(with_mask):
|
||||
engine = _mock_engine()
|
||||
|
||||
async def generate(*args, **kwargs):
|
||||
for position, tokens in enumerate(([10], [20, 30])):
|
||||
result = _make_request_output(
|
||||
"req-mask",
|
||||
token_ids=list(tokens),
|
||||
finish_reason="length" if position else None,
|
||||
finished=bool(position),
|
||||
)
|
||||
if with_mask:
|
||||
result.outputs[0].sampling_mask = SamplingMask(
|
||||
[[token, token + 1] for token in tokens]
|
||||
)
|
||||
yield result
|
||||
|
||||
engine.generate = MagicMock(side_effect=generate)
|
||||
serving = _build_serving_tokens(engine)
|
||||
response = await serving.serve_tokens(
|
||||
GenerateRequest(
|
||||
model=MODEL_NAME, token_ids=[1, 2, 3], sampling_params={}, stream=True
|
||||
)
|
||||
)
|
||||
chunks = _parse_sse_chunks([chunk async for chunk in response])
|
||||
choices = [
|
||||
choice
|
||||
for chunk in chunks
|
||||
if isinstance(chunk, dict)
|
||||
for choice in chunk.get("choices", [])
|
||||
]
|
||||
assert [choice["token_ids"] for choice in choices] == [[10], [20, 30]]
|
||||
for choice in choices:
|
||||
expected = (
|
||||
[[token, token + 1] for token in choice["token_ids"]] if with_mask else None
|
||||
)
|
||||
assert choice["sampling_mask"] == expected
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import os
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import regex as re
|
||||
import requests
|
||||
import torch
|
||||
from transformers import AutoModelForCausalLM
|
||||
|
||||
from tests.entrypoints.serve.dev.rlhf.conftest import (
|
||||
MODEL_NAME,
|
||||
gen,
|
||||
health,
|
||||
ok,
|
||||
server,
|
||||
)
|
||||
from tests.utils import multi_gpu_test
|
||||
from vllm.distributed.weight_transfer import (
|
||||
HTTPVLLMWeightSyncClient,
|
||||
ModuleSource,
|
||||
WeightTransferTrainerFactory,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.ipc_engine import IPCTrainerInitInfo
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
def weight_checker(url: str, action: str, baseline=None) -> requests.Response:
|
||||
body = {"action": action}
|
||||
if baseline is not None:
|
||||
body["baseline"] = baseline
|
||||
return requests.post(f"{url}/weight_checker", json=body, timeout=300)
|
||||
|
||||
|
||||
def checksums(url: str) -> dict[str, str]:
|
||||
response = weight_checker(url, "checksum")
|
||||
response.raise_for_status()
|
||||
return response.json()["checksums"]
|
||||
|
||||
|
||||
def ranks(keys) -> set[str]:
|
||||
return {key.rsplit(":", 1)[0] for key in keys}
|
||||
|
||||
|
||||
def reset_reload_and_compare(url: str) -> None:
|
||||
baseline = checksums(url)
|
||||
try:
|
||||
weight_checker(url, "reset").raise_for_status()
|
||||
comparison = weight_checker(url, "compare", baseline).json()
|
||||
assert ranks(comparison["mismatches"]) == ranks(baseline)
|
||||
finally:
|
||||
requests.post(
|
||||
f"{url}/collective_rpc", json={"method": "reload_weights"}, timeout=300
|
||||
).raise_for_status()
|
||||
assert weight_checker(url, "compare", baseline).json() == {
|
||||
"match": True,
|
||||
"mismatches": [],
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture(scope="class")
|
||||
def server_url():
|
||||
with server() as url:
|
||||
yield url
|
||||
|
||||
|
||||
class TestSingleGPU:
|
||||
@pytest.mark.parametrize(
|
||||
"payload",
|
||||
[{}, {"action": "frobnicate"}, {"action": "compare"}, [], None],
|
||||
)
|
||||
def test_invalid_request_returns_400(self, server_url, payload):
|
||||
response = requests.post(
|
||||
f"{server_url}/weight_checker", json=payload, timeout=10
|
||||
)
|
||||
assert response.status_code == 400, response.text
|
||||
assert health(server_url) == 200
|
||||
|
||||
def test_checksum_is_stable_and_compare_reports_differences(self, server_url):
|
||||
baseline = checksums(server_url)
|
||||
assert all(
|
||||
re.fullmatch(r"[0-9a-f]{64}", digest) for digest in baseline.values()
|
||||
)
|
||||
assert checksums(server_url) == baseline
|
||||
|
||||
changed, missing, *_ = baseline
|
||||
altered = {**baseline, changed: "0" * 64}
|
||||
del altered[missing]
|
||||
assert weight_checker(server_url, "compare", altered).json() == {
|
||||
"match": False,
|
||||
"mismatches": sorted([changed, missing]),
|
||||
}
|
||||
|
||||
def test_reset_changes_weights_and_reload_restores_them(self, server_url):
|
||||
reset_reload_and_compare(server_url)
|
||||
assert ok(gen(server_url))
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="IPC weight transfer uses CUDA IPC."
|
||||
)
|
||||
def test_ipc_weight_transfer_restores_reset_weights():
|
||||
"""The documented RL flow: reset, then transfer the checkpoint back."""
|
||||
if current_platform.is_xpu():
|
||||
pytest.skip("IPC weight transfer backend uses CUDA IPC handles")
|
||||
args = ["--weight-transfer-config", '{"backend": "ipc"}']
|
||||
with (
|
||||
patch.dict(os.environ, {"VLLM_ALLOW_INSECURE_SERIALIZATION": "1"}),
|
||||
server(extra_args=args, port=8772) as url,
|
||||
):
|
||||
baseline = checksums(url)
|
||||
weight_checker(url, "reset").raise_for_status()
|
||||
trainer_model = AutoModelForCausalLM.from_pretrained(
|
||||
MODEL_NAME, torch_dtype=torch.bfloat16
|
||||
).cuda()
|
||||
WeightTransferTrainerFactory.trainer_init(
|
||||
IPCTrainerInitInfo(rank=0),
|
||||
client=HTTPVLLMWeightSyncClient(url),
|
||||
source=ModuleSource(trainer_model),
|
||||
).send_weights()
|
||||
assert weight_checker(url, "compare", baseline).json() == {
|
||||
"match": True,
|
||||
"mismatches": [],
|
||||
}
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@pytest.mark.parametrize(
|
||||
"args, expected",
|
||||
[
|
||||
(["--data-parallel-size", "2"], {"dp0:pp0:pcp0:tp0", "dp1:pp0:pcp0:tp0"}),
|
||||
(["--tensor-parallel-size", "2"], {"dp0:pp0:pcp0:tp0", "dp0:pp0:pcp0:tp1"}),
|
||||
],
|
||||
)
|
||||
def test_checksum_and_reset_cover_every_worker(args, expected):
|
||||
with server(extra_args=args, port=8771, timeout=600) as url:
|
||||
assert ranks(checksums(url)) == expected
|
||||
reset_reload_and_compare(url)
|
||||
@@ -0,0 +1,93 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Weight-transfer metrics scraped from a real two-API-server deployment."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
import torch
|
||||
from prometheus_client.multiprocess import MultiProcessCollector
|
||||
from prometheus_client.parser import text_string_to_metric_families
|
||||
from transformers import AutoModelForCausalLM
|
||||
|
||||
from tests.entrypoints.serve.dev.rlhf.conftest import MODEL_NAME
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.distributed.weight_transfer import (
|
||||
HTTPVLLMWeightSyncClient,
|
||||
ModuleSource,
|
||||
WeightTransferTrainerFactory,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.ipc_engine import IPCTrainerInitInfo
|
||||
|
||||
DURATION = "vllm:rl_weight_update_operation_duration_seconds"
|
||||
IN_FLIGHT = "vllm:rl_weight_update_operations_in_flight"
|
||||
# The API servers accept from one shared listening socket, so the kernel may
|
||||
# hand one process most connections. Sync until both have recorded, within a cap.
|
||||
MIN_SYNCS = 8
|
||||
MAX_SYNCS = 64
|
||||
|
||||
|
||||
def scrape(url: str) -> dict[tuple[str, str], float]:
|
||||
text = requests.get(f"{url}/metrics", timeout=30).text
|
||||
return {
|
||||
(sample.name, sample.labels["operation"]): sample.value
|
||||
for family in text_string_to_metric_families(text)
|
||||
for sample in family.samples
|
||||
if sample.name in (f"{DURATION}_count", IN_FLIGHT)
|
||||
}
|
||||
|
||||
|
||||
def pids_that_recorded(multiproc_dir: Path) -> set[str]:
|
||||
return {
|
||||
path.stem.rsplit("_", 1)[1]
|
||||
for path in multiproc_dir.glob("histogram_*.db")
|
||||
if any(
|
||||
metric.name == DURATION
|
||||
for metric in MultiProcessCollector.merge([str(path)])
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
def test_weight_sync_metrics_aggregate_across_api_servers(tmp_path):
|
||||
args = [
|
||||
"--load-format",
|
||||
"dummy",
|
||||
"--enforce-eager",
|
||||
"--max-model-len",
|
||||
"512",
|
||||
"--gpu-memory-utilization",
|
||||
"0.5",
|
||||
"--api-server-count",
|
||||
"2",
|
||||
"--weight-transfer-config",
|
||||
'{"backend": "ipc"}',
|
||||
]
|
||||
env = {
|
||||
"VLLM_SERVER_DEV_MODE": "1",
|
||||
"VLLM_ALLOW_INSECURE_SERIALIZATION": "1",
|
||||
"PROMETHEUS_MULTIPROC_DIR": str(tmp_path),
|
||||
}
|
||||
with RemoteOpenAIServer(MODEL_NAME, args, env_dict=env) as server:
|
||||
url = server.url_root
|
||||
trainer = AutoModelForCausalLM.from_pretrained(
|
||||
MODEL_NAME, torch_dtype=torch.bfloat16
|
||||
).cuda()
|
||||
engine = WeightTransferTrainerFactory.trainer_init(
|
||||
IPCTrainerInitInfo(rank=0),
|
||||
client=HTTPVLLMWeightSyncClient(url),
|
||||
source=ModuleSource(trainer),
|
||||
)
|
||||
syncs = 0
|
||||
while syncs < MIN_SYNCS or (
|
||||
len(pids_that_recorded(tmp_path)) < 2 and syncs < MAX_SYNCS
|
||||
):
|
||||
engine.send_weights()
|
||||
syncs += 1
|
||||
metrics = scrape(url)
|
||||
|
||||
calls = {"init": 1, "start": syncs, "update": syncs, "finish": syncs}
|
||||
for operation, count in calls.items():
|
||||
assert metrics[(f"{DURATION}_count", operation)] == count, operation
|
||||
assert metrics[(IN_FLIGHT, operation)] == 0, operation
|
||||
# The counts above are sums over processes only if both servers recorded.
|
||||
assert len(pids_that_recorded(tmp_path)) == 2
|
||||
@@ -1,35 +1,209 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import asyncio
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
from prometheus_client import CollectorRegistry
|
||||
from prometheus_client.parser import text_string_to_metric_families
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.entrypoints.serve.dev.sleep import metrics as sleep_metrics
|
||||
from vllm.entrypoints.serve.dev.sleep.api_router import attach_router
|
||||
from vllm.entrypoints.serve.dev.sleep.metrics import SleepModeOperationMetrics
|
||||
from vllm.entrypoints.serve.exception_handling.register import init_exception_handler
|
||||
from vllm.entrypoints.serve.instrumentator.metrics import (
|
||||
attach_router as attach_metrics_router,
|
||||
)
|
||||
from vllm.v1.metrics import prometheus as prometheus_metrics
|
||||
|
||||
MODEL_NAME = "meta-llama/Llama-3.2-1B"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sleep_route_app(monkeypatch):
|
||||
app = FastAPI()
|
||||
app.state.args = SimpleNamespace(log_error_stack=False)
|
||||
app.state.engine_client = AsyncMock()
|
||||
metrics = SleepModeOperationMetrics(CollectorRegistry())
|
||||
monkeypatch.setattr(
|
||||
"vllm.entrypoints.serve.dev.sleep.api_router.sleep_mode_operation_metrics",
|
||||
lambda: metrics,
|
||||
)
|
||||
|
||||
attach_router(app)
|
||||
init_exception_handler(app)
|
||||
return app, metrics
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.parametrize("level", [0, 1, 2])
|
||||
def test_sleep_route_response_and_engine_arguments(sleep_route_app, level):
|
||||
app, _ = sleep_route_app
|
||||
with TestClient(app) as client:
|
||||
response = client.post("/sleep", params={"level": level})
|
||||
assert response.status_code == 200
|
||||
assert response.json() == {"status": "sleeping", "level": level}
|
||||
app.state.engine_client.sleep.assert_awaited_once_with(level, "abort")
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.parametrize(
|
||||
("query", "expected_param"),
|
||||
[
|
||||
("level=invalid", "query.level"),
|
||||
("level=-1", "query.level"),
|
||||
("level=3", "query.level"),
|
||||
("mode=invalid", "query.mode"),
|
||||
],
|
||||
)
|
||||
def test_sleep_route_rejects_invalid_query_before_dispatch(
|
||||
sleep_route_app, query, expected_param
|
||||
):
|
||||
app, metrics = sleep_route_app
|
||||
with TestClient(app) as client:
|
||||
response = client.post(f"/sleep?{query}")
|
||||
assert response.status_code == 400
|
||||
assert response.json()["error"]["param"] == expected_param
|
||||
app.state.engine_client.sleep.assert_not_awaited()
|
||||
assert list(metrics.duration.collect()[0].samples) == []
|
||||
assert list(metrics.in_flight.collect()[0].samples) == []
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.parametrize(
|
||||
("fully_awake", "tags", "expected"),
|
||||
[
|
||||
(True, "", {"status": "awake", "tags": None}),
|
||||
(False, "?tags=weights", {"status": "sleeping", "tags": ["weights"]}),
|
||||
],
|
||||
)
|
||||
def test_wake_route_maps_engine_result_without_state_query(
|
||||
sleep_route_app, fully_awake, tags, expected
|
||||
):
|
||||
app, _ = sleep_route_app
|
||||
app.state.engine_client.wake_up.return_value = fully_awake
|
||||
with TestClient(app) as client:
|
||||
response = client.post(f"/wake_up{tags}")
|
||||
assert response.status_code == 200
|
||||
assert response.json() == expected
|
||||
app.state.engine_client.wake_up.assert_awaited_once_with(expected["tags"])
|
||||
app.state.engine_client.is_sleeping.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.parametrize("fails", [False, True])
|
||||
def test_release_kv_cache_memory_route(fails):
|
||||
app = FastAPI()
|
||||
app.state.engine_client = AsyncMock()
|
||||
def test_release_kv_cache_memory_route(sleep_route_app, fails):
|
||||
app, metrics = sleep_route_app
|
||||
release = app.state.engine_client.release_kv_cache_memory
|
||||
if fails:
|
||||
release.side_effect = RuntimeError("requires a completed pause first")
|
||||
attach_router(app)
|
||||
|
||||
with TestClient(app, raise_server_exceptions=False) as client:
|
||||
response = client.post("/release_kv_cache_memory")
|
||||
|
||||
assert response.status_code == (500 if fails else 200)
|
||||
if not fails:
|
||||
assert response.json() == {"status": "kv_cache_released"}
|
||||
release.assert_awaited_once_with()
|
||||
assert metrics.in_flight.labels("release_kv_cache_memory")._value.get() == 0
|
||||
assert _duration_count(metrics, "release_kv_cache_memory") == 1
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.parametrize("operation", ["sleep", "release_kv_cache_memory", "wake"])
|
||||
@pytest.mark.parametrize("exception", [RuntimeError, asyncio.CancelledError])
|
||||
def test_sleep_mode_recorder_tracks_duration_and_cleans_up(operation, exception):
|
||||
metrics = SleepModeOperationMetrics(CollectorRegistry())
|
||||
with metrics.record(operation):
|
||||
assert metrics.in_flight.labels(operation)._value.get() == 1
|
||||
with pytest.raises(exception), metrics.record(operation):
|
||||
raise exception()
|
||||
assert metrics.in_flight.labels(operation)._value.get() == 0
|
||||
assert metrics.duration.labels(operation)._sum.get() >= 0
|
||||
assert _duration_count(metrics, operation) == 2
|
||||
|
||||
|
||||
def _duration_count(metrics, operation):
|
||||
return next(
|
||||
sample.value
|
||||
for sample in metrics.duration.collect()[0].samples
|
||||
if sample.name.endswith("_count") and sample.labels == {"operation": operation}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.parametrize(
|
||||
("path", "method", "operation", "status", "fails"),
|
||||
[
|
||||
("/sleep", "POST", "sleep", 200, False),
|
||||
("/release_kv_cache_memory", "POST", "release_kv_cache_memory", 200, False),
|
||||
("/wake_up", "POST", "wake", 200, False),
|
||||
("/is_sleeping", "GET", None, 200, False),
|
||||
("/sleep?level=3", "POST", None, 400, False),
|
||||
("/release_kv_cache_memory", "POST", "release_kv_cache_memory", 500, True),
|
||||
],
|
||||
)
|
||||
def test_sleep_routes_visible_on_production_metrics_endpoint(
|
||||
monkeypatch, path, method, operation, status, fails
|
||||
):
|
||||
registry = CollectorRegistry()
|
||||
monkeypatch.delenv("PROMETHEUS_MULTIPROC_DIR", raising=False)
|
||||
monkeypatch.setattr(sleep_metrics, "REGISTRY", registry)
|
||||
monkeypatch.setattr(sleep_metrics, "_metrics", None)
|
||||
monkeypatch.setattr(prometheus_metrics, "REGISTRY", registry)
|
||||
|
||||
app = FastAPI()
|
||||
app.state.args = SimpleNamespace(log_error_stack=False)
|
||||
app.state.engine_client = AsyncMock()
|
||||
app.state.engine_client.wake_up.return_value = True
|
||||
app.state.engine_client.is_sleeping.return_value = False
|
||||
if fails:
|
||||
app.state.engine_client.release_kv_cache_memory.side_effect = RuntimeError(
|
||||
"engine failed"
|
||||
)
|
||||
attach_router(app)
|
||||
init_exception_handler(app)
|
||||
attach_metrics_router(app)
|
||||
|
||||
with TestClient(app, raise_server_exceptions=False) as client:
|
||||
assert client.request(method, path).status_code == status
|
||||
response = client.get("/metrics")
|
||||
|
||||
assert response.status_code == 200
|
||||
samples = [
|
||||
sample
|
||||
for family in text_string_to_metric_families(response.text)
|
||||
for sample in family.samples
|
||||
]
|
||||
assert any(
|
||||
sample.name == "http_requests_total"
|
||||
and sample.labels
|
||||
== {
|
||||
"handler": path.split("?")[0],
|
||||
"method": method,
|
||||
"status": f"{status // 100}xx",
|
||||
}
|
||||
and sample.value == 1
|
||||
for sample in samples
|
||||
)
|
||||
assert not any("sleep_mode_operations_total" in sample.name for sample in samples)
|
||||
assert any(
|
||||
sample.name == "vllm:sleep_mode_operations_in_flight"
|
||||
and sample.labels == {"operation": operation}
|
||||
and sample.value == 0
|
||||
for sample in samples
|
||||
) == (operation is not None)
|
||||
assert any(
|
||||
sample.name == "vllm:sleep_mode_operation_duration_seconds_count"
|
||||
and sample.labels == {"operation": operation}
|
||||
and sample.value == 1
|
||||
for sample in samples
|
||||
) == (operation is not None)
|
||||
|
||||
|
||||
def test_sleep_mode():
|
||||
|
||||
@@ -29,7 +29,9 @@ LORA_UNLOADING_SUCCESS_MESSAGE = (
|
||||
)
|
||||
|
||||
|
||||
async def _async_serving_models_init() -> OpenAIServingModels:
|
||||
async def _async_serving_models_init_with_mock() -> tuple[
|
||||
OpenAIServingModels, MagicMock
|
||||
]:
|
||||
mock_engine_client = MagicMock(spec=EngineClient)
|
||||
# Set the max_model_len attribute to avoid missing attribute
|
||||
mock_model_config = MagicMock(spec=ModelConfig)
|
||||
@@ -45,6 +47,13 @@ async def _async_serving_models_init() -> OpenAIServingModels:
|
||||
)
|
||||
await serving_models.init_static_loras()
|
||||
|
||||
# The mock is returned separately: `engine_client` is typed as the protocol,
|
||||
# so assertion helpers are not visible through it.
|
||||
return serving_models, mock_engine_client
|
||||
|
||||
|
||||
async def _async_serving_models_init() -> OpenAIServingModels:
|
||||
serving_models, _ = await _async_serving_models_init_with_mock()
|
||||
return serving_models
|
||||
|
||||
|
||||
@@ -99,6 +108,22 @@ async def test_load_lora_adapter_duplicate():
|
||||
assert len(serving_models.lora_requests) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("load_inplace", [False, True])
|
||||
async def test_load_lora_adapter_base_model_name(load_inplace: bool):
|
||||
"""A LoRA named after a served model would shadow it for all requests."""
|
||||
serving_models, mock_engine_client = await _async_serving_models_init_with_mock()
|
||||
request = LoadLoRAAdapterRequest(
|
||||
lora_name=MODEL_NAME, lora_path="/path/to/adapter", load_inplace=load_inplace
|
||||
)
|
||||
response = await serving_models.load_lora_adapter(request)
|
||||
assert isinstance(response, ErrorResponse)
|
||||
assert response.error.type == "InvalidUserInput"
|
||||
assert response.error.code == HTTPStatus.BAD_REQUEST
|
||||
assert len(serving_models.lora_requests) == 0
|
||||
mock_engine_client.add_lora.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unload_lora_adapter_success():
|
||||
serving_models = await _async_serving_models_init()
|
||||
@@ -108,8 +133,8 @@ async def test_unload_lora_adapter_success():
|
||||
response = await serving_models.load_lora_adapter(request)
|
||||
assert len(serving_models.lora_requests) == 1
|
||||
|
||||
request = UnloadLoRAAdapterRequest(lora_name="adapter1")
|
||||
response = await serving_models.unload_lora_adapter(request)
|
||||
unload_request = UnloadLoRAAdapterRequest(lora_name="adapter1")
|
||||
response = await serving_models.unload_lora_adapter(unload_request)
|
||||
assert response == LORA_UNLOADING_SUCCESS_MESSAGE.format(lora_name="adapter1")
|
||||
assert len(serving_models.lora_requests) == 0
|
||||
|
||||
|
||||
@@ -84,7 +84,10 @@ def task_routes(request, monkeypatch) -> tuple[str, list[tuple[str, list[str]]]]
|
||||
|
||||
# Register routers for this specific task (development mode already enabled).
|
||||
register_api_routers(
|
||||
args, app, supported_tasks=(task,), model_config=MockModelConfig()
|
||||
args,
|
||||
app,
|
||||
supported_tasks=(task,),
|
||||
model_config=MockModelConfig(),
|
||||
)
|
||||
|
||||
routes = get_all_http_routes(app)
|
||||
|
||||
@@ -19,6 +19,7 @@ from vllm.entrypoints.serve.tokenize.api_router import attach_router
|
||||
from vllm.entrypoints.serve.tokenize.protocol import (
|
||||
TokenizeChatRequest,
|
||||
TokenizeCompletionRequest,
|
||||
TokenizeResponse,
|
||||
)
|
||||
from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
|
||||
from vllm.exceptions import VLLMNotFoundError, VLLMValidationError
|
||||
@@ -95,7 +96,7 @@ async def test_tokenize_chat_skips_mm_cache_for_renderer_only_path():
|
||||
mock_engine.renderer = MagicMock()
|
||||
|
||||
serving = _build_serving_tokenization(mock_engine)
|
||||
serving.online_renderer.preprocess_chat = AsyncMock(
|
||||
serving.online_renderer.preprocess_chat = AsyncMock( # type: ignore[method-assign]
|
||||
return_value=(
|
||||
[{"role": "user", "content": "Test"}],
|
||||
[{"prompt_token_ids": [1, 2, 3]}],
|
||||
@@ -109,7 +110,7 @@ async def test_tokenize_chat_skips_mm_cache_for_renderer_only_path():
|
||||
|
||||
response = await serving.create_tokenize(request, MagicMock(headers={}))
|
||||
|
||||
assert response.tokens == [1, 2, 3]
|
||||
assert isinstance(response, TokenizeResponse) and response.tokens == [1, 2, 3]
|
||||
assert (
|
||||
serving.online_renderer.preprocess_chat.call_args.kwargs["skip_mm_cache"]
|
||||
is True
|
||||
@@ -125,7 +126,7 @@ async def test_tokenize_completion_skips_mm_cache_for_renderer_only_path():
|
||||
mock_engine.renderer = MagicMock()
|
||||
|
||||
serving = _build_serving_tokenization(mock_engine)
|
||||
serving.online_renderer.preprocess_completion = AsyncMock(
|
||||
serving.online_renderer.preprocess_completion = AsyncMock( # type: ignore[method-assign]
|
||||
return_value=[{"prompt_token_ids": [1, 2, 3]}]
|
||||
)
|
||||
|
||||
@@ -136,7 +137,7 @@ async def test_tokenize_completion_skips_mm_cache_for_renderer_only_path():
|
||||
|
||||
response = await serving.create_tokenize(request, MagicMock(headers={}))
|
||||
|
||||
assert response.tokens == [1, 2, 3]
|
||||
assert isinstance(response, TokenizeResponse) and response.tokens == [1, 2, 3]
|
||||
assert (
|
||||
serving.online_renderer.preprocess_completion.call_args.kwargs["skip_mm_cache"]
|
||||
is True
|
||||
|
||||
@@ -101,7 +101,7 @@ async def test_tokenize_chat(
|
||||
if continue_final:
|
||||
conversation.append({"role": "assistant", "content": "Sure,"})
|
||||
|
||||
prompt = tokenizer.apply_chat_template(
|
||||
prompt = tokenizer.apply_chat_template( # type: ignore[call-arg]
|
||||
add_generation_prompt=add_generation,
|
||||
continue_final_message=continue_final,
|
||||
conversation=conversation,
|
||||
@@ -169,7 +169,7 @@ async def test_tokenize_chat_with_tools(
|
||||
if continue_final:
|
||||
conversation.append({"role": "assistant", "content": "Sure,"})
|
||||
|
||||
prompt = tokenizer.apply_chat_template(
|
||||
prompt = tokenizer.apply_chat_template( # type: ignore[call-arg]
|
||||
add_generation_prompt=add_generation,
|
||||
continue_final_message=continue_final,
|
||||
conversation=conversation,
|
||||
|
||||
@@ -18,7 +18,7 @@ def _cfg(tp=1, pp=1, dp=1, ep=False, digest="a3b21f94deadbeef"):
|
||||
enable_expert_parallel=ep,
|
||||
)
|
||||
)
|
||||
c.compute_hash = lambda: digest # type: ignore[attr-defined]
|
||||
c.compute_hash = lambda: digest
|
||||
return c
|
||||
|
||||
|
||||
@@ -59,6 +59,7 @@ def test_get_respects_set_default():
|
||||
|
||||
fp.set_default_fingerprint_mode("hash")
|
||||
hashed = fp.get_system_fingerprint(cfg)
|
||||
assert hashed is not None
|
||||
assert hashed != full
|
||||
assert "tp8" not in hashed
|
||||
|
||||
@@ -72,5 +73,7 @@ def test_get_respects_set_default():
|
||||
def test_compute_hash_failure_does_not_raise():
|
||||
cfg = _cfg()
|
||||
cfg.compute_hash = lambda: (_ for _ in ()).throw(RuntimeError("boom"))
|
||||
assert fp.build_system_fingerprint(cfg, "full").endswith("-nohash")
|
||||
assert fp.build_system_fingerprint(cfg, "hash").endswith("-nohash")
|
||||
full_fp = fp.build_system_fingerprint(cfg, "full")
|
||||
assert full_fp is not None and full_fp.endswith("-nohash")
|
||||
hash_fp = fp.build_system_fingerprint(cfg, "hash")
|
||||
assert hash_fp is not None and hash_fp.endswith("-nohash")
|
||||
|
||||
@@ -47,7 +47,7 @@ def test_sanitize_handles_falsy_inputs() -> None:
|
||||
assert _sanitize_transcription_user_text("") == ""
|
||||
# The dataclass default for ``response_prefix`` is the empty string;
|
||||
# the sanitizer must accept that without exception or extra work.
|
||||
assert _sanitize_transcription_user_text(None) == "" # type: ignore[arg-type]
|
||||
assert _sanitize_transcription_user_text(None) == ""
|
||||
|
||||
|
||||
def test_sanitize_is_idempotent() -> None:
|
||||
|
||||
@@ -117,10 +117,7 @@ MESSAGES_INVALID_CALL = [
|
||||
|
||||
# Expected outputs
|
||||
FUNC_CALC = "calculator"
|
||||
FUNC_ARGS_CALC = '{"expression":"123 + 456"}'
|
||||
|
||||
FUNC_TIME = "get_time"
|
||||
FUNC_ARGS_TIME = '{"city": "New York"}'
|
||||
|
||||
|
||||
# ==========================================================
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Claude Code's attribution header in chat completions system messages."""
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.chat_utils import _parse_chat_message_content
|
||||
|
||||
HEADER = (
|
||||
"x-anthropic-billing-header: cc_version=2.1.220.300; "
|
||||
"cc_entrypoint=claude-desktop; cch=7adab;"
|
||||
)
|
||||
|
||||
|
||||
def _parse(message, content_format="openai"):
|
||||
tracker = MagicMock()
|
||||
tracker.create_parser.return_value.mm_placeholder_storage.return_value = {}
|
||||
return _parse_chat_message_content(
|
||||
message, tracker, content_format, interleave_strings=False
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
@pytest.mark.parametrize(
|
||||
"header",
|
||||
[
|
||||
HEADER,
|
||||
"x-anthropic-billing-header: cc_version=2.1.286.99b; cc_entrypoint=sdk-cli;",
|
||||
],
|
||||
)
|
||||
def test_system_header_part_is_dropped(header):
|
||||
(msg,) = _parse(
|
||||
{
|
||||
"role": "system",
|
||||
"content": [
|
||||
{"type": "text", "text": header},
|
||||
{"type": "text", "text": "You are a Claude agent."},
|
||||
],
|
||||
}
|
||||
)
|
||||
assert msg["content"] == [{"type": "text", "text": "You are a Claude agent."}]
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_system_header_line_is_stripped_from_string():
|
||||
(msg,) = _parse(
|
||||
{"role": "system", "content": HEADER + "\nYou are a Claude agent."},
|
||||
content_format="string",
|
||||
)
|
||||
assert msg["content"] == "You are a Claude agent."
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_requests_differing_only_in_cch_parse_identically():
|
||||
def system(cch):
|
||||
return {
|
||||
"role": "system",
|
||||
"content": HEADER.replace("7adab", cch) + "\nYou are a Claude agent.",
|
||||
}
|
||||
|
||||
assert _parse(system("7adab")) == _parse(system("986dd"))
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
@pytest.mark.parametrize(
|
||||
"message",
|
||||
[
|
||||
{"role": "system", "content": "You are a Claude agent."},
|
||||
{"role": "system", "content": "Quote: " + HEADER},
|
||||
{"role": "user", "content": HEADER + "Hello"},
|
||||
],
|
||||
)
|
||||
def test_other_content_is_unchanged(message):
|
||||
(msg,) = _parse(message, content_format="string")
|
||||
assert msg["content"] == message["content"]
|
||||
@@ -17,6 +17,7 @@ from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.chat_utils import (
|
||||
MEDIA_CONNECTOR_REGISTRY,
|
||||
AsyncMultiModalItemTracker,
|
||||
ChatCompletionMessageParam,
|
||||
ConversationMessage,
|
||||
_load_embeds_dict,
|
||||
_parse_metadata_array,
|
||||
@@ -769,7 +770,9 @@ async def test_text_only_chat_does_not_initialize_media_connector(
|
||||
):
|
||||
load_connector = MagicMock()
|
||||
monkeypatch.setattr(MEDIA_CONNECTOR_REGISTRY, "load", load_connector)
|
||||
messages = [{"role": "user", "content": "Who are you?"}]
|
||||
messages: list[ChatCompletionMessageParam] = [
|
||||
{"role": "user", "content": "Who are you?"}
|
||||
]
|
||||
|
||||
parse_chat_messages(
|
||||
messages,
|
||||
@@ -2457,7 +2460,7 @@ def test_parse_chat_messages_image_vision_chunk(
|
||||
kimi_k2_5_model_config,
|
||||
image_url,
|
||||
):
|
||||
messages = [
|
||||
messages: list[ChatCompletionMessageParam] = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
@@ -2493,7 +2496,7 @@ def test_parse_chat_messages_video_vision_chunk(
|
||||
kimi_k2_5_model_config,
|
||||
video_url,
|
||||
):
|
||||
messages = [
|
||||
messages: list[ChatCompletionMessageParam] = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
@@ -2530,7 +2533,7 @@ def test_parse_chat_messages_image_vision_chunk_with_uuid(
|
||||
image_url,
|
||||
):
|
||||
image_uuid = "image_123"
|
||||
messages = [
|
||||
messages: list[ChatCompletionMessageParam] = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
@@ -2568,7 +2571,7 @@ def test_parse_chat_messages_video_vision_chunk_with_uuid(
|
||||
video_url,
|
||||
):
|
||||
video_uuid = "video_456"
|
||||
messages = [
|
||||
messages: list[ChatCompletionMessageParam] = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
@@ -2606,7 +2609,7 @@ def test_parse_chat_messages_mixed_vision_chunk(
|
||||
image_url,
|
||||
video_url,
|
||||
):
|
||||
messages = [
|
||||
messages: list[ChatCompletionMessageParam] = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
@@ -2655,7 +2658,7 @@ def test_parse_chat_messages_mixed_vision_chunk_with_uuid(
|
||||
):
|
||||
image_uuid = "image_123"
|
||||
video_uuid = "video_456"
|
||||
messages = [
|
||||
messages: list[ChatCompletionMessageParam] = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
@@ -2707,7 +2710,7 @@ async def test_parse_chat_messages_mixed_vision_chunk_async(
|
||||
image_url,
|
||||
video_url,
|
||||
):
|
||||
messages = [
|
||||
messages: list[ChatCompletionMessageParam] = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
@@ -2757,7 +2760,7 @@ async def test_parse_chat_messages_mixed_vision_chunk_with_uuid_async(
|
||||
):
|
||||
image_uuid = "image_123"
|
||||
video_uuid = "video_456"
|
||||
messages = [
|
||||
messages: list[ChatCompletionMessageParam] = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
@@ -2808,7 +2811,7 @@ async def test_parse_chat_messages_image_vision_chunk_async(
|
||||
kimi_k2_5_model_config,
|
||||
image_url,
|
||||
):
|
||||
messages = [
|
||||
messages: list[ChatCompletionMessageParam] = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
@@ -2845,7 +2848,7 @@ async def test_parse_chat_messages_video_vision_chunk_async(
|
||||
kimi_k2_5_model_config,
|
||||
video_url,
|
||||
):
|
||||
messages = [
|
||||
messages: list[ChatCompletionMessageParam] = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
@@ -2883,7 +2886,7 @@ async def test_parse_chat_messages_image_vision_chunk_with_uuid_async(
|
||||
image_url,
|
||||
):
|
||||
image_uuid = "image_123"
|
||||
messages = [
|
||||
messages: list[ChatCompletionMessageParam] = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
@@ -2922,7 +2925,7 @@ async def test_parse_chat_messages_video_vision_chunk_with_uuid_async(
|
||||
video_url,
|
||||
):
|
||||
video_uuid = "video_456"
|
||||
messages = [
|
||||
messages: list[ChatCompletionMessageParam] = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
@@ -2997,8 +3000,7 @@ async def test_resolve_items_runs_modalities_concurrently_and_preserves_order():
|
||||
finally:
|
||||
active_fetches -= 1
|
||||
|
||||
tracker = AsyncMultiModalItemTracker(MagicMock())
|
||||
tracker._model_config.is_multimodal_model = True
|
||||
tracker = AsyncMultiModalItemTracker(MagicMock(is_multimodal_model=True))
|
||||
tracker.__dict__["mm_processor"] = MagicMock()
|
||||
tracker._items_by_modality["video"] = [
|
||||
lambda: _fetch("video-0", 0.02),
|
||||
@@ -3192,5 +3194,5 @@ def test_validate_chat_template_rejects_invalid_type():
|
||||
with pytest.raises(
|
||||
VLLMValidationError, match="not a valid chat template type"
|
||||
) as exc_info:
|
||||
validate_chat_template(123) # type: ignore[arg-type]
|
||||
validate_chat_template(123)
|
||||
assert exc_info.value.parameter == "chat_template"
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Malformed bodies on RL dev routes are 400s that never reach the engine."""
|
||||
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from vllm.entrypoints.serve.dev.rlhf.api_router import router as rlhf_router
|
||||
from vllm.entrypoints.serve.dev.rpc.api_router import router as rpc_router
|
||||
from vllm.entrypoints.serve.exception_handling.register import init_exception_handler
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
|
||||
class Engine:
|
||||
def __init__(self):
|
||||
self.calls: list[str] = []
|
||||
|
||||
def __getattr__(self, name):
|
||||
async def call(*args, **kwargs):
|
||||
self.calls.append(name)
|
||||
|
||||
return call
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client():
|
||||
app = FastAPI()
|
||||
app.include_router(rlhf_router)
|
||||
app.include_router(rpc_router)
|
||||
init_exception_handler(app)
|
||||
app.state.engine_client = Engine()
|
||||
app.state.args = SimpleNamespace(log_error_stack=False)
|
||||
return TestClient(app)
|
||||
|
||||
|
||||
def post(client, path, body):
|
||||
return client.post(
|
||||
path, content=json.dumps(body), headers={"content-type": "application/json"}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"path",
|
||||
[
|
||||
"/abort_requests",
|
||||
"/init_weight_transfer_engine",
|
||||
"/update_weights",
|
||||
"/collective_rpc",
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("body", [[], 1, "x", None])
|
||||
def test_non_object_body_is_rejected(client, path, body):
|
||||
assert post(client, path, body).status_code == 400
|
||||
assert client.app.state.engine_client.calls == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"path,body",
|
||||
[
|
||||
("/init_weight_transfer_engine", {}),
|
||||
("/init_weight_transfer_engine", {"init_info": []}),
|
||||
("/init_weight_transfer_engine", {"init_info": 1}),
|
||||
("/update_weights", {}),
|
||||
("/update_weights", {"update_info": 1}),
|
||||
("/update_weights", {"update_info": [1, 2]}),
|
||||
("/update_weights", {"update_info": "x"}),
|
||||
],
|
||||
)
|
||||
def test_invalid_field_is_rejected_before_the_engine(client, path, body):
|
||||
# Reaching the engine would also abort an in-progress weight update.
|
||||
assert post(client, path, body).status_code == 400
|
||||
assert client.app.state.engine_client.calls == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"path,body,engine_call",
|
||||
[
|
||||
("/abort_requests", {"request_ids": ["a"]}, "abort"),
|
||||
(
|
||||
"/init_weight_transfer_engine",
|
||||
{"init_info": {}},
|
||||
"init_weight_transfer_engine",
|
||||
),
|
||||
("/update_weights", {"update_info": {"names": []}}, "update_weights"),
|
||||
("/update_weights", {"update_info": [{}, {}]}, "update_weights"),
|
||||
("/collective_rpc", {"method": "m"}, "collective_rpc"),
|
||||
],
|
||||
)
|
||||
def test_valid_body_reaches_the_engine(client, path, body, engine_call):
|
||||
assert post(client, path, body).status_code == 200
|
||||
assert client.app.state.engine_client.calls == [engine_call]
|
||||
@@ -0,0 +1,188 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import asyncio
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from contextlib import nullcontext
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
from prometheus_client import CollectorRegistry
|
||||
|
||||
from vllm.entrypoints.serve.dev.rlhf import api_router
|
||||
from vllm.entrypoints.serve.dev.rlhf import metrics as rlhf_metrics
|
||||
from vllm.entrypoints.serve.dev.rlhf.metrics import WeightOperationMetrics
|
||||
from vllm.entrypoints.serve.exception_handling.register import init_exception_handler
|
||||
from vllm.v1.metrics.prometheus import get_prometheus_registry
|
||||
|
||||
pytestmark = [pytest.mark.cpu_test, pytest.mark.skip_global_cleanup]
|
||||
|
||||
DURATION = "vllm:rl_weight_update_operation_duration_seconds_count"
|
||||
IN_FLIGHT = "vllm:rl_weight_update_operations_in_flight"
|
||||
|
||||
|
||||
def sample(registry, name, operation):
|
||||
return registry.get_sample_value(name, {"operation": operation})
|
||||
|
||||
|
||||
class Request:
|
||||
def __init__(self, engine):
|
||||
self.app = SimpleNamespace(state=SimpleNamespace(engine_client=engine))
|
||||
|
||||
|
||||
class Engine:
|
||||
"""Holds every engine call open until the test releases it."""
|
||||
|
||||
def __init__(self):
|
||||
self.started = asyncio.Event()
|
||||
self.release = asyncio.Event()
|
||||
|
||||
def __getattr__(self, name):
|
||||
async def call(*args, **kwargs):
|
||||
self.started.set()
|
||||
await self.release.wait()
|
||||
|
||||
return call
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def registry(monkeypatch):
|
||||
registry = CollectorRegistry()
|
||||
monkeypatch.setattr(rlhf_metrics, "_metrics", WeightOperationMetrics(registry))
|
||||
return registry
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fail", [False, True])
|
||||
def test_record_releases_in_flight_and_observes_duration(registry, fail):
|
||||
metrics = rlhf_metrics.weight_operation_metrics()
|
||||
with (
|
||||
pytest.raises(RuntimeError) if fail else nullcontext(),
|
||||
metrics.record("update"),
|
||||
):
|
||||
assert sample(registry, IN_FLIGHT, "update") == 1
|
||||
if fail:
|
||||
raise RuntimeError
|
||||
assert sample(registry, IN_FLIGHT, "update") == 0
|
||||
assert sample(registry, DURATION, "update") == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"operation,route,body",
|
||||
[
|
||||
("init", api_router.init_weight_transfer_engine, {"init_info": {}}),
|
||||
("start", api_router.start_weight_update, None),
|
||||
("start_draft", api_router.start_draft_weight_update, None),
|
||||
("update", api_router.update_weights, {"update_info": {}}),
|
||||
("finish", api_router.finish_weight_update, None),
|
||||
],
|
||||
)
|
||||
async def test_route_records_its_operation(registry, operation, route, body):
|
||||
engine = Engine()
|
||||
task = asyncio.create_task(route(Request(engine), **(body or {})))
|
||||
await engine.started.wait()
|
||||
assert sample(registry, IN_FLIGHT, operation) == 1
|
||||
engine.release.set()
|
||||
await task
|
||||
assert sample(registry, IN_FLIGHT, operation) == 0
|
||||
assert sample(registry, DURATION, operation) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cancelled_route_releases_in_flight(registry):
|
||||
engine = Engine()
|
||||
task = asyncio.create_task(
|
||||
api_router.update_weights(Request(engine), update_info={})
|
||||
)
|
||||
await engine.started.wait()
|
||||
task.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await task
|
||||
assert sample(registry, IN_FLIGHT, "update") == 0
|
||||
assert sample(registry, DURATION, "update") == 1
|
||||
|
||||
|
||||
HOLD_OPERATION = """
|
||||
import sys
|
||||
from vllm.entrypoints.serve.dev.rlhf.metrics import weight_operation_metrics
|
||||
with weight_operation_metrics().record("update"):
|
||||
print("entered", flush=True)
|
||||
sys.stdin.readline()
|
||||
"""
|
||||
|
||||
|
||||
def test_in_flight_is_summed_across_api_server_processes(tmp_path, monkeypatch):
|
||||
"""Two API-server processes each hold an operation; /metrics sums them."""
|
||||
monkeypatch.setenv("PROMETHEUS_MULTIPROC_DIR", str(tmp_path))
|
||||
servers = [
|
||||
subprocess.Popen(
|
||||
[sys.executable, "-c", HOLD_OPERATION],
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
text=True,
|
||||
env=os.environ.copy(),
|
||||
)
|
||||
for _ in range(2)
|
||||
]
|
||||
try:
|
||||
for server in servers:
|
||||
assert server.stdout is not None
|
||||
assert server.stdout.readline().strip() == "entered"
|
||||
assert sample(get_prometheus_registry(), IN_FLIGHT, "update") == 2
|
||||
finally:
|
||||
for server in servers:
|
||||
server.communicate("\n", timeout=60)
|
||||
|
||||
scraped = get_prometheus_registry()
|
||||
assert sample(scraped, IN_FLIGHT, "update") == 0
|
||||
assert sample(scraped, DURATION, "update") == 2
|
||||
|
||||
|
||||
def test_rejected_request_is_not_recorded(registry):
|
||||
app = FastAPI()
|
||||
app.include_router(api_router.router)
|
||||
init_exception_handler(app)
|
||||
app.state.engine_client = Engine()
|
||||
app.state.args = SimpleNamespace(log_error_stack=False)
|
||||
response = TestClient(app).post("/update_weights", json={})
|
||||
assert response.status_code == 400
|
||||
assert sample(registry, DURATION, "update") is None
|
||||
|
||||
|
||||
def test_default_recorder_is_created_lazily_on_the_default_registry():
|
||||
"""The default recorder registers on the process registry only when first used.
|
||||
|
||||
Runs in a fresh interpreter: the registry is process-global, so a route test
|
||||
elsewhere in the session may already have registered these collectors, and
|
||||
"not registered yet" would then be unobservable.
|
||||
"""
|
||||
probe = """
|
||||
from prometheus_client import REGISTRY
|
||||
|
||||
from vllm.entrypoints.serve.dev.rlhf import metrics as rlhf_metrics
|
||||
|
||||
NAMES = [
|
||||
"vllm:rl_weight_update_operation_duration_seconds",
|
||||
"vllm:rl_weight_update_operations_in_flight",
|
||||
]
|
||||
assert rlhf_metrics._metrics is None
|
||||
assert not set(NAMES) & set(REGISTRY._names_to_collectors)
|
||||
|
||||
metrics = rlhf_metrics.weight_operation_metrics()
|
||||
assert REGISTRY._names_to_collectors[NAMES[0]] is metrics.duration
|
||||
assert REGISTRY._names_to_collectors[NAMES[1]] is metrics.in_flight
|
||||
assert rlhf_metrics.weight_operation_metrics() is metrics
|
||||
print("lazy registration ok")
|
||||
"""
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-c", probe],
|
||||
cwd=Path(__file__).parents[3],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert "lazy registration ok" in result.stdout
|
||||
@@ -0,0 +1,17 @@
|
||||
model_name: "deepseek-ai/DeepSeek-R1"
|
||||
accuracy_threshold: 0.95
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
startup_max_wait_seconds: 1800
|
||||
env:
|
||||
VLLM_ROCM_USE_AITER: "1"
|
||||
VLLM_ROCM_USE_AITER_MLA: "1"
|
||||
VLLM_ROCM_USE_AITER_MOE: "1"
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 4096
|
||||
--data-parallel-size 4
|
||||
--enable-expert-parallel
|
||||
--attention-backend ROCM_AITER_MLA
|
||||
--moe-backend aiter
|
||||
--speculative-config '{"method":"mtp","num_speculative_tokens":3}'
|
||||
@@ -0,0 +1,17 @@
|
||||
model_name: "deepseek-ai/DeepSeek-R1"
|
||||
accuracy_threshold: 0.95
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
startup_max_wait_seconds: 1800
|
||||
env:
|
||||
VLLM_ROCM_USE_AITER: "1"
|
||||
VLLM_ROCM_USE_AITER_MLA: "1"
|
||||
VLLM_ROCM_USE_AITER_MOE: "1"
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 4096
|
||||
--tensor-parallel-size 4
|
||||
--enable-expert-parallel
|
||||
--attention-backend ROCM_AITER_MLA
|
||||
--moe-backend aiter
|
||||
--speculative-config '{"method":"mtp","num_speculative_tokens":3}'
|
||||
@@ -0,0 +1,17 @@
|
||||
model_name: "deepseek-ai/DeepSeek-V3.2"
|
||||
accuracy_threshold: 0.95
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
startup_max_wait_seconds: 1200
|
||||
env:
|
||||
VLLM_ROCM_USE_AITER: "1"
|
||||
VLLM_ROCM_USE_AITER_MLA: "1"
|
||||
VLLM_ROCM_USE_AITER_MOE: "1"
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 4096
|
||||
--data-parallel-size 4
|
||||
--enable-expert-parallel
|
||||
--attention-backend ROCM_AITER_MLA_SPARSE
|
||||
--moe-backend aiter
|
||||
--speculative-config '{"method":"mtp","num_speculative_tokens":3}'
|
||||
@@ -0,0 +1,17 @@
|
||||
model_name: "deepseek-ai/DeepSeek-V3.2"
|
||||
accuracy_threshold: 0.95
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
startup_max_wait_seconds: 1200
|
||||
env:
|
||||
VLLM_ROCM_USE_AITER: "1"
|
||||
VLLM_ROCM_USE_AITER_MLA: "1"
|
||||
VLLM_ROCM_USE_AITER_MOE: "1"
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 4096
|
||||
--tensor-parallel-size 4
|
||||
--enable-expert-parallel
|
||||
--attention-backend ROCM_AITER_MLA_SPARSE
|
||||
--moe-backend aiter
|
||||
--speculative-config '{"method":"mtp","num_speculative_tokens":3}'
|
||||
@@ -0,0 +1,15 @@
|
||||
model_name: "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16"
|
||||
accuracy_threshold: 0.93
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
startup_max_wait_seconds: 1200
|
||||
env:
|
||||
VLLM_ROCM_USE_AITER: "1"
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 4096
|
||||
--tensor-parallel-size 4
|
||||
--enable-expert-parallel
|
||||
--mamba-backend triton
|
||||
--moe-backend triton
|
||||
--speculative-config '{"method":"mtp","num_speculative_tokens":5}'
|
||||
@@ -0,0 +1,15 @@
|
||||
model_name: "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8"
|
||||
accuracy_threshold: 0.93
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
startup_max_wait_seconds: 1200
|
||||
env:
|
||||
VLLM_ROCM_USE_AITER: "1"
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 4096
|
||||
--tensor-parallel-size 4
|
||||
--enable-expert-parallel
|
||||
--mamba-backend triton
|
||||
--moe-backend triton
|
||||
--speculative-config '{"method":"mtp","num_speculative_tokens":5}'
|
||||
@@ -0,0 +1,18 @@
|
||||
model_name: "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4"
|
||||
accuracy_threshold: 0.93
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
max_concurrency: 16
|
||||
rocm_request_timeout_seconds: 1800
|
||||
min_acceptance_length: 2.0
|
||||
startup_max_wait_seconds: 1200
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 4096
|
||||
--max-num-seqs 16
|
||||
--max-num-batched-tokens 32768
|
||||
--gpu-memory-utilization 0.5
|
||||
--tensor-parallel-size 2
|
||||
--enable-expert-parallel
|
||||
--mamba-backend triton
|
||||
--speculative-config '{"method":"mtp","num_speculative_tokens":5}'
|
||||
@@ -0,0 +1,17 @@
|
||||
model_name: "nm-testing/Qwen3-Next-80B-A3B-Instruct-NVFP4"
|
||||
accuracy_threshold: 0.75
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
max_concurrency: 16
|
||||
rocm_request_timeout_seconds: 1800
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 4096
|
||||
--max-num-seqs 16
|
||||
--max-num-batched-tokens 32768
|
||||
--gpu-memory-utilization 0.5
|
||||
--tensor-parallel-size 2
|
||||
--enable-expert-parallel
|
||||
--all2all-backend allgather_reducescatter
|
||||
--attention-backend TRITON_ATTN
|
||||
--no-async-scheduling
|
||||
@@ -0,0 +1,19 @@
|
||||
model_name: "nvidia/Qwen3-Next-80B-A3B-Instruct-NVFP4"
|
||||
accuracy_threshold: 0.85
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
max_concurrency: 16
|
||||
rocm_request_timeout_seconds: 1800
|
||||
min_acceptance_length: 1.5
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 4096
|
||||
--max-num-seqs 16
|
||||
--max-num-batched-tokens 32768
|
||||
--gpu-memory-utilization 0.5
|
||||
--tensor-parallel-size 2
|
||||
--enable-expert-parallel
|
||||
--all2all-backend allgather_reducescatter
|
||||
--attention-backend TRITON_ATTN
|
||||
--no-async-scheduling
|
||||
--speculative-config '{"method":"qwen3_next_mtp","num_speculative_tokens":1}'
|
||||
@@ -0,0 +1,5 @@
|
||||
# Exact B200 compressed-tensors checkpoint; it does not contain MTP weights.
|
||||
Qwen3-Next-80B-A3B-NVFP4-EP2-CT-MI355.yaml
|
||||
# MTP-capable ModelOpt checkpoint exercises NVFP4 EP2 with MTP1.
|
||||
Qwen3-Next-80B-A3B-NVFP4-EP2-MTP-MI355.yaml
|
||||
Nemotron-3-Super-120B-A12B-NVFP4-MI355.yaml
|
||||
@@ -1,2 +1,5 @@
|
||||
Qwen3-0.6B-FP8.yaml
|
||||
Qwen3-1.7B-MXFP4.yaml
|
||||
Qwen3-30B-A3B-NVFP4.yaml
|
||||
Laguna-XS.2-NVFP4.yaml
|
||||
Laguna-XS-2.1-INT4.yaml
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
DeepSeek-R1-DP_MI355.yaml
|
||||
@@ -0,0 +1 @@
|
||||
DeepSeek-R1-TP_MI355.yaml
|
||||
@@ -0,0 +1 @@
|
||||
DeepSeek-V3.2-DP_MI355.yaml
|
||||
@@ -0,0 +1 @@
|
||||
DeepSeek-V3.2-TP_MI355.yaml
|
||||
@@ -0,0 +1,2 @@
|
||||
Nemotron-3-Super-120B-A12B-BF16_MI355.yaml
|
||||
Nemotron-3-Super-120B-A12B-FP8_MI355.yaml
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user