Add pydocstyle to the ruff rules (#52136)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Signed-off-by: Harry Mellor <hej.mellor@gmail.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-09-16 16:44:29 +00:00
committed by GitHub
co-authored by Claude Opus 5
parent 804e5377aa
commit 8c1557a79c
2002 changed files with 15024 additions and 14269 deletions
@@ -89,7 +89,7 @@ TARGET_TEST_SUITE = _resolve_target_test_suite()
condition=current_platform.is_rocm(),
)
def test_vllm_gc_ed():
"""Verify vllm instance is GC'ed when it is deleted"""
"""Verify vllm instance is GC'ed when it is deleted."""
llm = LLM("hmellor/tiny-random-LlamaForCausalLM")
weak_llm = weakref.ref(llm)
del llm
+3 -3
View File
@@ -51,8 +51,7 @@ def _wake_up_with_poisoned_mappings(allocator, byte_value: int = 0xA5) -> None:
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
def test_python_error():
"""
Test if Python error occurs when there's low-level
"""Test if Python error occurs when there's low-level
error happening from the C++ side.
"""
allocator = get_mem_allocator_instance()
@@ -113,7 +112,8 @@ def test_basic_cumem():
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
def test_discard_tags():
"""Test that discard(tags) selectively frees GPU memory for specific
tags while keeping other tags mapped and usable."""
tags while keeping other tags mapped and usable.
"""
allocator = get_mem_allocator_instance()
with allocator.use_memory_pool("weights"):
+12 -6
View File
@@ -13,7 +13,8 @@ from vllm.benchmarks.datasets import BFCLDataset, get_samples
def _patch_hf_api(side_effect):
"""Return a patch context that swaps `hf_api()` to a stub whose
`.hf_hub_download` attribute uses `side_effect`."""
`.hf_hub_download` attribute uses `side_effect`.
"""
fake_api = MagicMock()
fake_api.hf_hub_download.side_effect = side_effect
return patch("vllm.benchmarks.datasets.datasets.hf_api", return_value=fake_api)
@@ -115,7 +116,8 @@ def test_bfcl_dataset_translates_schema_and_attaches_tools(
) -> None:
"""BFCLDataset should translate schemas to OpenAI tool format, set
`messages` directly on SampleRequest, and attach tools/tool_choice via
request_overrides."""
request_overrides.
"""
paths = _write_fake_files(tmp_path)
def fake_download(_repo, filename, **_kwargs):
@@ -166,7 +168,8 @@ def test_bfcl_dataset_missing_category_raises_clear_error(
hf_tokenizer: PreTrainedTokenizerBase,
) -> None:
"""A typo'd category should produce an actionable ValueError, not an
opaque huggingface_hub exception."""
opaque huggingface_hub exception.
"""
from huggingface_hub.errors import EntryNotFoundError
args = _args_for_bfcl(categories=["simpl"]) # typo
@@ -184,7 +187,8 @@ def test_bfcl_dataset_missing_category_raises_clear_error(
@pytest.mark.benchmark
def test_chat_backend_uses_messages_field_when_set() -> None:
"""When RequestFuncInput.chat_messages is set, the chat backend must use
it verbatim and skip default content construction from `prompt`."""
it verbatim and skip default content construction from `prompt`.
"""
import asyncio
from vllm.benchmarks.lib.endpoint_request_func import (
@@ -244,7 +248,8 @@ def test_chat_backend_uses_messages_field_when_set() -> None:
def test_bfcl_prompt_len_includes_tools(tmp_path: Path) -> None:
"""prompt_len must reflect tokens from both messages *and* tool schemas,
so percentile buckets and input-distribution summaries aren't biased
low for tool-heavy traffic."""
low for tool-heavy traffic.
"""
paths = _write_fake_files(tmp_path)
def fake_download(_repo, filename, **_kwargs):
@@ -291,7 +296,8 @@ def test_bfcl_prompt_len_falls_back_when_tokenizer_rejects_tools(
tmp_path: Path,
) -> None:
"""Older tokenizers don't accept tools=; fallback must still produce a
non-zero prompt_len without crashing."""
non-zero prompt_len without crashing.
"""
paths = _write_fake_files(tmp_path)
def fake_download(_repo, filename, **_kwargs):
@@ -12,7 +12,8 @@ from vllm.benchmarks.datasets import get_samples
class _RecordingTokenizer:
"""Minimal tokenizer stub that records the kwargs forwarded to
apply_chat_template, so we can assert chat_template_kwargs propagation
without loading a real model/template."""
without loading a real model/template.
"""
def __init__(self) -> None:
self.captured_kwargs: dict | None = None
@@ -228,7 +228,7 @@ def test_decode_span_identity_is_exact(
api_url: str,
chat: bool,
) -> None:
"""latency - ttft must equal sum(itl) exactly.
"""Latency - ttft must equal sum(itl) exactly.
On a scripted clock this can only drift if a code path reads the clock an
extra time or attributes a read to the wrong event.
+6 -3
View File
@@ -10,7 +10,8 @@ from vllm.tokenizers import TokenizerLike
class _FakeTokenizer(TokenizerLike):
"""Minimal tokenizer implementing the TokenizerLike protocol
for testing get_sampling_params."""
for testing get_sampling_params.
"""
def __init__(self, vocab_size: int = 1000, num_special_tokens: int = 0) -> None:
self._vocab_size = vocab_size
@@ -171,7 +172,8 @@ class TestGetSamplingParams:
def test_output_low_clamped_to_one(self):
"""Even with a high ratio that would push output_low to 0,
the function clamps it to 1."""
the function clamps it to 1.
"""
rng = np.random.default_rng(0)
# output_len=1, ratio=0.99 → floor(1*0.01)=0, should clamp to 1
_, output_lens, _ = get_sampling_params(
@@ -233,7 +235,8 @@ class TestGetSamplingParams:
def test_input_len_zero_with_special_tokens(self):
"""input_len < num_special_tokens → real_input_len = 0, which is fine
(range [0, 0])."""
(range [0, 0]).
"""
rng = np.random.default_rng(0)
input_lens, _, _ = get_sampling_params(
rng, 5, 0.0, 5, 50, self._tok(num_special=10)
+2 -1
View File
@@ -52,7 +52,8 @@ def _write_dataset(path: Path) -> None:
def _args(dataset_path: str) -> argparse.Namespace:
"""Reproduce the argparse.Namespace that serve.py builds from the
failing command, including skip_tokenizer_init=True."""
failing command, including skip_tokenizer_init=True.
"""
return argparse.Namespace(
# dataset
dataset_name="custom",
+1 -3
View File
@@ -1,8 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
These envs only work for a small part of the tests, fix what you need!
"""
"""These envs only work for a small part of the tests, fix what you need!"""
import os
from collections.abc import Callable
+2 -4
View File
@@ -29,8 +29,7 @@ logger = init_logger("vllm.tests.compile.backend")
class LazyInitPass(InductorPass):
"""
If there's a pass that we want to initialize lazily in a test,
"""If there's a pass that we want to initialize lazily in a test,
we can wrap it in LazyInitPass, which will initialize the pass when invoked
and then immediately invoke it.
"""
@@ -45,8 +44,7 @@ class LazyInitPass(InductorPass):
class TestBackend:
"""
This class provides a simple Inductor backend that can be used for testing.
"""This class provides a simple Inductor backend that can be used for testing.
It takes a list of custom passes and runs them after Inductor's passes.
It also saves the graph before and after the custom passes for inspection.
+2 -4
View File
@@ -11,8 +11,7 @@ from vllm.platforms.interface import DeviceCapability
@pytest.fixture
def mock_cuda_platform():
"""
Fixture that returns a factory for creating mocked CUDA platforms.
"""Fixture that returns a factory for creating mocked CUDA platforms.
Usage:
def test_something(mock_cuda_platform):
@@ -51,8 +50,7 @@ def mock_cuda_platform():
@pytest.fixture
def mock_xpu_platform():
"""
Fixture that returns a factory for creating mocked XPU platforms.
"""Fixture that returns a factory for creating mocked XPU platforms.
Usage:
def test_something(mock_xpu_platform):
@@ -1,11 +1,10 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
WARNING: This test runs in both single-node (4 GPUs) and multi-node
(2 node with 2 GPUs each) modes. If the test only uses 2 GPUs, it is
important to set the distributed backend to "mp" to avoid Ray scheduling
all workers in a node other than the head node, which can cause the test
to fail.
"""WARNING: This test runs in both single-node (4 GPUs) and multi-node
(2 node with 2 GPUs each) modes. If the test only uses 2 GPUs, it is
important to set the distributed backend to "mp" to avoid Ray scheduling
all workers in a node other than the head node, which can cause the test
to fail.
"""
import json
@@ -17,8 +17,7 @@ from vllm.v1.attention.backends.registry import AttentionBackendEnum
@contextlib.contextmanager
def temporary_environ(env_vars):
"""
Temporarily set environment variables and restore them afterward.
"""Temporarily set environment variables and restore them afterward.
We have to do this vs monkeypatch because monkeypatch doesn't work
with "module" scoped fixtures.
"""
@@ -125,8 +124,7 @@ def llm_pair(request):
indirect=True,
)
class TestFullCUDAGraph:
"""
Use a class such that an llm pair is constructed once for all
"""Use a class such that an llm pair is constructed once for all
batch_size/max_tokens combinations and released immediately after.
Module-scope fixtures would stick around the whole time,
@@ -149,11 +147,9 @@ class TestFullCUDAGraph:
],
)
def test_full_cudagraph(self, batch_size, max_tokens, llm_pair: tuple[LLM, LLM]):
"""
Test various batch sizes and max_tokens to ensure that the
"""Test various batch sizes and max_tokens to ensure that the
full cudagraph compilation works for padded cases too.
"""
full_cudagraph_llm, piecewise_llm = llm_pair
prompts = ["the quick brown fox"] * batch_size
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Test (piecewise) compilation with a simple model where multiple submodules
"""Test (piecewise) compilation with a simple model where multiple submodules
are compiled and graph captured separately.
"""
+2 -4
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Test the piecewise compilation with a simple model so that we
"""Test the piecewise compilation with a simple model so that we
can exactly calculate the expected output and side effects.
"""
@@ -52,8 +51,7 @@ class SillyModel(nn.Module):
self.intermediate_unbacked = intermediate_unbacked
def forward(self, x: torch.Tensor) -> torch.Tensor:
"""
Overall effect:
"""Overall effect:
x = 3 * x + 19
global_counter += 2
"""
+1 -2
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Test the piecewise compilation with a simple model, comparing the output
"""Test the piecewise compilation with a simple model, comparing the output
with and without the piecewise compilation.
This is a tractable model, the weights and computation are specially designed
@@ -56,10 +56,7 @@ class TestMMRSModel(torch.nn.Module):
torch.nn.init.normal_(self.gate_proj, std=0.02)
def forward(self, hidden_states):
"""
Forward pass implementing the mm + reduce scatter in the FX graph
"""
"""Forward pass implementing the mm + reduce scatter in the FX graph."""
# Reshape input
view = hidden_states.reshape(-1, self.hidden_size)
@@ -88,9 +85,7 @@ class TestAGMMModel(torch.nn.Module):
torch.nn.init.normal_(self.weight, std=0.02)
def forward(self, hidden_states):
"""
Forward pass implementing the mm + all gather in the FX graph
"""
"""Forward pass implementing the mm + all gather in the FX graph."""
# Reshape input
view = hidden_states.reshape(-1, self.hidden_size)
all_gather = tensor_model_parallel_all_gather(view, dim=0)
@@ -122,10 +117,7 @@ class _BaseScaledMMModel(torch.nn.Module):
class TestScaledMMRSModel(_BaseScaledMMModel):
def forward(self, input: torch.Tensor):
"""
Forward pass implementing the scaled_mm + reduce scatter in the FX graph
"""
"""Forward pass implementing the scaled_mm + reduce scatter in the FX graph."""
fp8_input = input.to(FP8_DTYPE)
scale_a = torch.ones(input.shape[0], 1, dtype=torch.float32)
scaled_mm = torch._scaled_mm(
@@ -147,9 +139,7 @@ class TestScaledMMRSModel(_BaseScaledMMModel):
class TestAGScaledMMModel(_BaseScaledMMModel):
def forward(self, input: torch.Tensor):
"""
Forward pass implementing the all gather + scaled_mm in the FX graph
"""
"""Forward pass implementing the all gather + scaled_mm in the FX graph."""
# Reshape input
fp8_input = input.to(FP8_DTYPE)
all_gather = tensor_model_parallel_all_gather(fp8_input, dim=0)
@@ -173,8 +163,7 @@ class TestAGScaledMMModel(_BaseScaledMMModel):
class TestCutlassScaledMMRSModel(_BaseScaledMMModel):
def forward(self, input: torch.Tensor):
"""
Forward pass implementing the cutlass_scaled_mm + reduce scatter
"""Forward pass implementing the cutlass_scaled_mm + reduce scatter
in the FX graph
"""
@@ -200,8 +189,7 @@ class TestCutlassScaledMMRSModel(_BaseScaledMMModel):
class TestAGCutlassScaledMMModel(_BaseScaledMMModel):
def forward(self, input: torch.Tensor):
"""
Forward pass implementing the all gather + cutlass_scaled_mm
"""Forward pass implementing the all gather + cutlass_scaled_mm
in the FX graph
"""
# Reshape input
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Comprehensive tests for UnsafeCloneEliminationPass.
"""Comprehensive tests for UnsafeCloneEliminationPass.
This test suite exercises all possible valid FX graph patterns involving clones:
1. Clone with no users (dead code)
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Tests for IR inplace functionalization pass integration.
"""Tests for IR inplace functionalization pass integration.
This test suite verifies that the inplace functionalization pass, lowering pass,
and clone cleanup pass work together correctly with donated buffer tracking.
@@ -108,8 +107,7 @@ class MixedModel(nn.Module):
class ModelWithTritonAfterMaybeInplace(nn.Module):
"""
Model using maybe_inplace followed by a Triton kernel.
"""Model using maybe_inplace followed by a Triton kernel.
Test clone elimination can handle Triton in the graph
"""
@@ -410,8 +408,7 @@ def with_dyn_arg(fn: Callable, arg_index: int, dim_index: int):
reason="Only test on cuda and rocm platform",
)
def test_piecewise_compilation_with_donated_buffers(monkeypatch, fresh_vllm_cache):
"""
Test piecewise compilation with donated buffers across graph splits.
"""Test piecewise compilation with donated buffers across graph splits.
Utilizes a custom splitting op. Uses fresh cache to avoid compilation caching.
"""
torch.set_default_device(current_platform.device_type)
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Unit tests for the DoubleQuant fan-out variants registered by
"""Unit tests for the DoubleQuant fan-out variants registered by
``RocmAiterRMSNormQuantFusionPass``.
Both variants target a 1-to-2 fan-out where one ``rms_norm`` output feeds
@@ -90,8 +89,7 @@ def test_double_aiter_rms_fp8_group_quant_fusion(
model_cls: type[torch.nn.Module],
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""
Both fan-out shapes (with and without an intermediate view) must fuse
"""Both fan-out shapes (with and without an intermediate view) must fuse
into ``rocm_aiter_rmsnorm_fp8_group_quant``: the no-view shape via
``DoubleAiterRMSFp8GroupQuantPattern`` and the viewed shape via the
new ``DoubleAiterRMSFp8GroupQuantViewPattern`` sibling.
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Unit test for the MLADualRMSNormFusionPass.
"""Unit test for the MLADualRMSNormFusionPass.
The pass fuses paired q/kv RMS norms in MLA attention into a single
fused_mla_dual_rms_norm op backed by AITER's fused_qk_rmsnorm kernel.
@@ -38,13 +37,12 @@ FP8_DTYPE = current_platform.fp8_dtype()
class MLADualRMSNormTestModel(torch.nn.Module):
"""
Minimal model reproducing the MLA dual RMS norm pattern:
linear -> split([q_dim, kv_dim])
+-- q_c (getitem 0) -> rms_norm(q_w, eps) -> linear
+-- kv_lora (getitem 1) -> split([kv_c_dim, k_pe_dim])
+-- kv_c (getitem 0) -> rms_norm(kv_w, eps)
+-- k_pe
"""Minimal model reproducing the MLA dual RMS norm pattern:
linear -> split([q_dim, kv_dim])
+-- q_c (getitem 0) -> rms_norm(q_w, eps) -> linear
+-- kv_lora (getitem 1) -> split([kv_c_dim, k_pe_dim])
+-- kv_c (getitem 0) -> rms_norm(kv_w, eps)
+-- k_pe
"""
def __init__(
@@ -155,13 +153,12 @@ def test_fuse_mla_dual_rms_norm(
class MLADualRMSNormFp8PerTokenTestModel(torch.nn.Module):
"""
Minimal model reproducing the FP8 MLA attention path with *per-token* quant:
linear -> split([q_dim, kv_dim])
+-- q_c (getitem 0) -> rocm_aiter_rmsnorm_fused_dynamic_quant -> dequant
+-- kv_lora (getitem 1) -> split([kv_c_dim, k_pe_dim])
+-- kv_c (getitem 0) -> rms_norm (bf16)
+-- k_pe
"""Minimal model reproducing the FP8 MLA attention path with *per-token* quant:
linear -> split([q_dim, kv_dim])
+-- q_c (getitem 0) -> rocm_aiter_rmsnorm_fused_dynamic_quant -> dequant
+-- kv_lora (getitem 1) -> split([kv_c_dim, k_pe_dim])
+-- kv_c (getitem 0) -> rms_norm (bf16)
+-- k_pe
"""
def __init__(
+2 -1
View File
@@ -732,7 +732,8 @@ def test_aiter_fusion_rmsnorm_gated_quant_no_gdn_layers(
monkeypatch: pytest.MonkeyPatch,
):
"""Verify that without GDN layers in static_forward_context,
the gated pattern is not registered and no matches occur."""
the gated pattern is not registered and no matches occur.
"""
group_shape = GroupShape(1, 128)
vllm_config = VllmConfig(
model_config=ModelConfig(dtype=dtype),
+1 -2
View File
@@ -107,7 +107,6 @@ class AttentionQuantPatternModel(torch.nn.Module):
def build_attn_metadata(self, batch_size: int) -> AttentionMetadata:
"""Initialize attention metadata."""
# TODO (Rohan138) reuse utils from vllm/v1/worker/gpu/attn_utils.py
# Create common attn metadata
@@ -304,7 +303,7 @@ def test_attention_quant_pattern(
dist_init,
disable_vllm_compile_cache,
):
"""Test AttentionStaticQuantPattern fusion pass"""
"""Test AttentionStaticQuantPattern fusion pass."""
if backend == AttentionBackendEnum.FLASHINFER and (
not current_platform.is_device_capability((10, 0)) or not has_flashinfer()
):
@@ -141,7 +141,6 @@ class MLAAttentionQuantPatternModel(torch.nn.Module):
the full output[:num_actual_toks] buffer after both forward_mha and
forward_mqa have written their results.
"""
batch_spec = BatchSpec(seq_lens=[1] * batch_size, query_lens=[1] * batch_size)
common_attn_metadata = create_common_attn_metadata(
batch_spec, self.block_size, self.device, arange_block_indices=True
@@ -408,7 +407,7 @@ def test_mla_attention_quant_pattern(
dist_init,
disable_vllm_compile_cache,
):
"""Test MLA AttentionQuantPattern fusion pass"""
"""Test MLA AttentionQuantPattern fusion pass."""
if (
model_class is TestMLAAttentionNvfp4QuantPatternModel
and not is_nvfp4_supported()
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Tests for manual fusion via maybe_fused_act_quant.
"""Tests for manual fusion via maybe_fused_act_quant.
Tests all fusion paths in _FUSED_ACT_QUANT:
- kFp8StaticTensorSym: all platforms
@@ -15,7 +15,8 @@ DEVICE_TYPE = current_platform.device_type
class SplitCoalescingModel(torch.nn.Module):
"""Model with 3 separate split_with_sizes calls on the same input,
simulating the B200+FP8 graph where CSE fails to merge them."""
simulating the B200+FP8 graph where CSE fails to merge them.
"""
def __init__(self, q_size: int, kv_size: int) -> None:
super().__init__()
+5 -7
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Shared PyTorch custom silly attention for compilation tests.
"""Shared PyTorch custom silly attention for compilation tests.
Centralizes custom operation definitions to avoid duplicate registrations.
"""
@@ -21,12 +20,12 @@ _global_counter = 0
def get_global_counter():
"""Get the current global counter value"""
"""Get the current global counter value."""
return _global_counter
def reset_global_counter():
"""Reset the global counter to 0"""
"""Reset the global counter to 0."""
global _global_counter
_global_counter = 0
@@ -34,8 +33,7 @@ def reset_global_counter():
def silly_attention(
q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, out: torch.Tensor
) -> None:
"""
Unified attention implementation that depends on
"""Unified attention implementation that depends on
all inputs and affects the output.
Always increments a global counter that tests can use or ignore.
"""
@@ -51,7 +49,7 @@ def silly_attention(
def silly_attention_fake(
q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, out: torch.Tensor
) -> None:
"""Fake implementation for testing"""
"""Fake implementation for testing."""
return
+7 -12
View File
@@ -268,8 +268,7 @@ def test_save_and_load_slice(monkeypatch: pytest.MonkeyPatch):
def test_cache_load_returns_tuple_consistency_tuple_output(
monkeypatch: pytest.MonkeyPatch,
):
"""
Test that cache loading correctly handles models that return tuples.
"""Test that cache loading correctly handles models that return tuples.
This verifies that when a model returns a tuple of tensors, the output
type is preserved as a tuple between fresh compilation and cache load.
@@ -335,8 +334,7 @@ def test_cache_load_returns_tuple_consistency_tuple_output(
@pytest.mark.skipif(not is_torch_equal_or_newer("2.10.0"), reason="requires torch 2.10")
def test_shape_env(monkeypatch: pytest.MonkeyPatch):
"""
Test that the shape environment is correctly serialized and preserved
"""Test that the shape environment is correctly serialized and preserved
when loading from cache.
"""
with monkeypatch.context() as m:
@@ -374,8 +372,7 @@ def test_shape_env(monkeypatch: pytest.MonkeyPatch):
def test_partition_wrapper_applied_on_aot_load(
monkeypatch: pytest.MonkeyPatch, vllm_tmp_cache: Path, mocker
):
"""
Test that partition wrappers are applied when loading AOT cached functions.
"""Test that partition wrappers are applied when loading AOT cached functions.
This test verifies the fix for GitHub issue #31439 where AOT compile
caused 2x latency regression when use_inductor_graph_partition=True.
@@ -498,14 +495,12 @@ def test_standalone_compile_correctness():
@pytest.mark.skipif(not is_torch_equal_or_newer("2.10.0"), reason="requires torch 2.10")
@create_new_process_for_each_test("spawn")
def test_gpt2_cache_hit(monkeypatch: pytest.MonkeyPatch):
"""
Test that compiling gpt2 twice results in a cache hit.
"""Test that compiling gpt2 twice results in a cache hit.
Counter values are read from the EngineCore subprocess via
``LLM.collective_rpc`` so the test works under default V1
multiprocessing (no shared memory between test and engine).
"""
from vllm import LLM
def _snap(self):
@@ -641,7 +636,7 @@ class TestStandaloneCompiledArtifacts:
@patch("torch._inductor.standalone_compile.AOTCompiledArtifact.deserialize")
def test_load_all_success(self, mock_deserialize):
"""Test successful loading of all artifacts"""
"""Test successful loading of all artifacts."""
cache = StandaloneCompiledArtifacts()
mock_artifact1 = Mock()
@@ -658,7 +653,7 @@ class TestStandaloneCompiledArtifacts:
@patch("torch._inductor.standalone_compile.AOTCompiledArtifact.deserialize")
def test_load_all_already_loaded(self, mock_deserialize):
"""Test that load_all skips if already loaded"""
"""Test that load_all skips if already loaded."""
cache = StandaloneCompiledArtifacts()
mock_artifact = Mock()
@@ -671,7 +666,7 @@ class TestStandaloneCompiledArtifacts:
@patch("torch._inductor.standalone_compile.AOTCompiledArtifact.deserialize")
def test_get_loaded_artifact(self, mock_deserialize):
"""Test retrieving loaded artifacts"""
"""Test retrieving loaded artifacts."""
cache = StandaloneCompiledArtifacts()
mock_artifact = Mock()
+20 -10
View File
@@ -77,7 +77,8 @@ def test_non_primitive_kwargs_lifted_to_consts(
into the namespace, so ``device`` is unbound. The fix collects such
objects into ``__vllm_consts__`` and references them by index. The
unqualified ``device(type=...)`` form must never appear in the
generated source."""
generated source.
"""
split_gm = _trace_and_split(model_fn, (x,), split_ops)
code, submod_names, consts = generate_execution_code(split_gm)
@@ -98,7 +99,8 @@ def test_dtype_singleton_deduped(x: torch.Tensor) -> None:
"""``torch.float16`` is a process-wide singleton, so two ops referring
to it in the traced graph share a single consts slot via ``id()``-based
dedup. Distinct expressions (``x.to(...)`` vs ``(x*2).to(...)``) ensure
the tracer can't CSE the two ops into a single node."""
the tracer can't CSE the two ops into a single node.
"""
def model_fn(x: torch.Tensor) -> torch.Tensor:
return x.to(dtype=torch.float16) + (x * 2).to(dtype=torch.float16)
@@ -147,7 +149,8 @@ def test_distinct_dtypes_get_distinct_slots(x: torch.Tensor) -> None:
def test_consts_ordering_deterministic(x: torch.Tensor) -> None:
"""Two independent traces of the same model must produce equal consts
lists *in the same order*. Cache artifacts identify const slots by
index, so a non-deterministic order would invalidate cached code."""
index, so a non-deterministic order would invalidate cached code.
"""
def model_fn(x: torch.Tensor) -> torch.Tensor:
# Multiple distinct non-primitives encountered in a fixed graph order.
@@ -185,7 +188,8 @@ def test_consts_shared_across_split_submods(x: torch.Tensor) -> None:
The function below splits into three inlined submods, two of which
independently reference ``torch.float16``. The shared ``const_index``
threaded through recursive ``generate_execution_code_with_name`` calls
must collapse the dtype to a single slot used from both submods."""
must collapse the dtype to a single slot used from both submods.
"""
def model_fn(x: torch.Tensor) -> torch.Tensor:
a = x.to(dtype=torch.float16) # submod_0: _to_copy(fp16)
@@ -230,7 +234,8 @@ def test_non_graphmodule_submod_uses_indexed_callable(x: torch.Tensor) -> None:
"""When a child of split_gm is *not* a ``torch.fx.GraphModule`` — as
happens in production once ``PiecewiseBackend`` replaces submods —
codegen emits ``__vllm_submods__[idx](...)`` instead of inlining, and
the runtime callable is bound from ``submod_callables``."""
the runtime callable is bound from ``submod_callables``.
"""
def model_fn(x: torch.Tensor) -> torch.Tensor:
return x.relu().sigmoid()
@@ -280,7 +285,8 @@ def test_non_graphmodule_submod_uses_indexed_callable(x: torch.Tensor) -> None:
def test_getitem_in_stitching_graph(x: torch.Tensor) -> None:
"""``operator.getitem`` on submod tuple returns is the ``call_function``
special case at codegen.py — emitted as ``name = source[index]``
rather than a function call."""
rather than a function call.
"""
def model_fn(x: torch.Tensor) -> torch.Tensor:
return x.relu().sigmoid()
@@ -298,7 +304,8 @@ def test_getitem_in_stitching_graph(x: torch.Tensor) -> None:
def test_del_emitted_for_intermediate_values(x: torch.Tensor) -> None:
"""The codegen schedules ``del`` after a value's last use to free
memory early. Multi-submod splits naturally have intermediates whose
last use is not the output node."""
last use is not the output node.
"""
def model_fn(x: torch.Tensor) -> torch.Tensor:
return x.relu().sigmoid().tanh()
@@ -318,7 +325,8 @@ def test_with_submod_false_rejects_call_module() -> None:
"""``generate_execution_code_with_name(with_submod=False)`` is the
recursive entry for inlining a GraphModule into its parent. It must
refuse a graph that itself contains ``call_module`` nodes — the parent
is responsible for handling those."""
is responsible for handling those.
"""
g = fx.Graph()
x_node = g.placeholder("x")
root = torch.nn.Module()
@@ -335,7 +343,8 @@ def test_node_ref_recurses_through_containers() -> None:
"""``_node_ref`` is the recursive walker that lifts non-primitives
nested inside list/tuple/dict args. Real aten ops rarely produce such
structures, but the path is needed for DTensor placement lists and
other future cases — unit-test the walker directly."""
other future cases — unit-test the walker directly.
"""
consts: list = []
const_index: dict[int, int] = {}
cpu = torch.device("cpu")
@@ -359,7 +368,8 @@ def test_node_ref_recurses_through_containers() -> None:
def test_legacy_code_without_consts() -> None:
"""``compile_execution_fn(consts=None)`` must still load code that has
no ``__vllm_consts__`` reference, so older serialized cache artifacts
keep working."""
keep working.
"""
# Pre-consts codegen: no __vllm_consts__ reference, only torch/operator.
legacy_code = (
"import torch\n"
+4 -2
View File
@@ -129,7 +129,8 @@ def test_compile_config_get_compile_ranges():
class PostGradStaticShapeChecker(InductorPass):
"""Asserts that compile_sizes entries produce graphs with fully concrete
(non-symbolic) shapes, and compile_ranges entries have symbolic shapes."""
(non-symbolic) shapes, and compile_ranges entries have symbolic shapes.
"""
def __init__(self):
self.num_static_calls = 0
@@ -170,7 +171,8 @@ class PostGradStaticShapeChecker(InductorPass):
def test_compile_sizes_produce_static_shapes(disable_vllm_compile_cache):
"""Verify that compile_sizes entries are compiled with fully concrete
shapes (no SymInts), while compile_ranges entries retain dynamic shapes."""
shapes (no SymInts), while compile_ranges entries retain dynamic shapes.
"""
checker = PostGradStaticShapeChecker()
torch.set_default_device("cuda")
vllm_config = VllmConfig(
+6 -3
View File
@@ -1229,7 +1229,8 @@ def test_compile_sizes_padding_validation():
def test_inductor_asserts_default_disabled(monkeypatch):
"""Test that inductor runtime asserts are disabled by default
(INFO logging level) on torch < 2.12."""
(INFO logging level) on torch < 2.12.
"""
monkeypatch.setenv("VLLM_LOGGING_LEVEL", "INFO")
import importlib
@@ -1247,7 +1248,8 @@ def test_inductor_asserts_default_disabled(monkeypatch):
def test_inductor_asserts_enabled_in_debug(monkeypatch):
"""Test that VLLM_LOGGING_LEVEL=DEBUG enables inductor runtime asserts
on torch < 2.12."""
on torch < 2.12.
"""
monkeypatch.setenv("VLLM_LOGGING_LEVEL", "DEBUG")
import importlib
@@ -1283,7 +1285,8 @@ def test_get_inductor_factors_includes_configs():
def test_inductor_asserts_user_override(monkeypatch):
"""Test that explicit inductor_compile_config overrides the
debug-logging default."""
debug-logging default.
"""
monkeypatch.setenv("VLLM_LOGGING_LEVEL", "INFO")
import importlib
@@ -22,7 +22,7 @@ from vllm.utils.torch_utils import is_torch_equal_or_newer
def get_test_models():
"""Get list of models to test based on PyTorch version"""
"""Get list of models to test based on PyTorch version."""
models = [
"Qwen/Qwen3-0.6B",
"openai-community/gpt2",
@@ -172,7 +172,6 @@ def test_model_specialization_with_evaluate_guards(
"""Test that evaluate_guards correctly detects shape specialization
violations.
"""
if (
use_aot_compile
and dynamic_shapes_type == DynamicShapesType.BACKED
+14 -29
View File
@@ -24,8 +24,7 @@ DEVICE_TYPE = current_platform.device_type
def test_getitem_moved_to_producer_subgraph():
"""
Test that getitem operations are moved to the same subgraph as their input,
"""Test that getitem operations are moved to the same subgraph as their input,
preventing tuple inputs to submodules.
"""
@@ -79,8 +78,7 @@ def test_getitem_moved_to_producer_subgraph():
def test_no_tuple_inputs_with_multiple_consumers():
"""
Test that when a tuple is consumed by multiple split operations,
"""Test that when a tuple is consumed by multiple split operations,
getitem operations are properly moved to avoid tuple inputs.
"""
@@ -138,13 +136,10 @@ def test_no_tuple_inputs_with_multiple_consumers():
def test_consecutive_ops_in_split():
"""
Test that consecutive splitting operations are grouped into the same subgraph
"""
"""Test that consecutive splitting operations are grouped into the same subgraph."""
def model_fn(x: torch.Tensor) -> torch.Tensor:
"""
Define a simple model where consecutive operations create opportunities
"""Define a simple model where consecutive operations create opportunities
for splitting subgraphs.
"""
# Apply silly attention followed by consecutive operations
@@ -211,8 +206,7 @@ def _subgraphs_with_empty_nodes(split_items, *, is_splitting_graph):
def test_empty_only_partition_stays_separate_after_splitting_predecessor():
"""
Empty-only subgraphs should not be merged when the only predecessor is
"""Empty-only subgraphs should not be merged when the only predecessor is
a splitting-op subgraph.
"""
@@ -248,8 +242,7 @@ def test_empty_only_partition_stays_separate_after_splitting_predecessor():
def test_empty_only_partition_is_merged():
"""
Empty-only subgraphs should still be merged when a non-splitting predecessor
"""Empty-only subgraphs should still be merged when a non-splitting predecessor
exists. The merged empty node must remain outside splitting-op subgraphs.
"""
@@ -294,8 +287,7 @@ def test_empty_only_partition_is_merged():
def test_builtin_empty_only_partition_is_merged():
"""
In Dynamo graphs, torch.empty/empty_like may appear as builtin call targets
"""In Dynamo graphs, torch.empty/empty_like may appear as builtin call targets
(not aten OpOverload). Ensure empty-only partitions are still merged.
"""
@@ -340,8 +332,7 @@ def test_builtin_empty_only_partition_is_merged():
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_sym_size_whole_shape_boundary():
"""
Test that using x.size() (whole shape) across a split boundary can be
"""Test that using x.size() (whole shape) across a split boundary can be
compiled by standalone_compile.
The dynamo graph looks like:
@@ -400,8 +391,7 @@ def test_sym_size_whole_shape_boundary():
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_symint_crosses_split_boundary():
"""
Test that SymInt placeholders from torch.compile + mark_dynamic
"""Test that SymInt placeholders from torch.compile + mark_dynamic
cross split boundaries safely via split_module's natural threading.
SymInt values are threaded through subgraphs by split_module and
@@ -459,8 +449,7 @@ def test_symint_crosses_split_boundary():
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_shape_boundary_standalone_compile():
"""
Repro for the original production bug:
"""Repro for the original production bug:
AssertionError: out_spec mismatch
TreeSpec(tuple, None, [*, *, TreeSpec(Size, None, [*, *]), *])
@@ -529,8 +518,7 @@ def test_shape_boundary_standalone_compile():
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_size_used_in_multiple_consumer_subgraphs():
"""
Validates that x.size() (whole shape) used by multiple downstream subgraphs
"""Validates that x.size() (whole shape) used by multiple downstream subgraphs
does not cause torch.Size to cross split boundaries.
Model:
@@ -584,8 +572,7 @@ def test_size_used_in_multiple_consumer_subgraphs():
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_sym_size_metadata_propagated():
"""
Validates that new sym_size.int nodes created by the pre-pass have
"""Validates that new sym_size.int nodes created by the pre-pass have
example_value metadata set. Without it, placeholder metadata in consumer
subgraphs would be None, breaking any code that dynamically builds
example inputs from metadata (e.g. standalone_compile per-submodule).
@@ -634,8 +621,7 @@ def test_sym_size_metadata_propagated():
def test_decompose_size_with_getitem_user():
"""
Regression test: _decompose_size_nodes must handle getitem users of size()
"""Regression test: _decompose_size_nodes must handle getitem users of size()
correctly.
When a graph contains x.shape[i], it can appear as:
@@ -704,8 +690,7 @@ def test_decompose_size_with_getitem_user():
def test_decompose_size_leaves_scalar_size_with_dim():
"""
Regression test: _decompose_size_nodes must leave x.size(dim) alone.
"""Regression test: _decompose_size_nodes must leave x.size(dim) alone.
x.size() returns a torch.Size tuple that can't cross split boundaries and
must be decomposed. x.size(dim), however, already returns a scalar
@@ -236,7 +236,8 @@ class TestInductorFallbackFastPath:
def test_inner_set_membership_still_works_for_standard_ops(self):
"""The patch must not break Inductor's existing fallback decisions
for non-vllm ops such as ``torchvision::roi_align``."""
for non-vllm ops such as ``torchvision::roi_align``.
"""
import torch._inductor.lowering as _lowering
allow_list = _lowering.FALLBACK_ALLOW_LIST
+1
View File
@@ -60,6 +60,7 @@ class TraceStructuredCapture:
Args:
event_type: The event type to filter by (e.g., "artifact", "graph_dump")
name_pattern: Regex pattern to match against the artifact name
"""
regex = re.compile(name_pattern)
return [
+1 -1
View File
@@ -40,7 +40,7 @@ class DummyLogprobsMode(Enum):
def test_hash_factors_deterministic():
"""Test that hash_factors produces consistent SHA-256 hashes"""
"""Test that hash_factors produces consistent SHA-256 hashes."""
factors = {"a": 1, "b": "test"}
hash1 = hash_factors(factors)
hash2 = hash_factors(factors)
+4 -2
View File
@@ -372,7 +372,8 @@ class _HeterogeneousConfig(PretrainedConfig):
"""A heterogeneous config with no convertor of its own.
Mirrors the parts vLLM uses: per-layer configs are shallow copies with the
varying attributes applied and heterogeneity stripped, so they do not recurse."""
varying attributes applied and heterogeneity stripped, so they do not recurse.
"""
is_heterogeneous = True
@@ -471,7 +472,8 @@ def test_deepseek_v4_convertor_splits_vision_architecture():
"""The vision checkpoint shares model_type/architectures with the
text-only DeepSeek-V4; the convertor routes it to the VL wrapper class
by rewriting hf_config.architectures (model-class resolution reads the
raw attribute)."""
raw attribute).
"""
from vllm.transformers_utils.configs.deepseek_v4 import DeepseekV4Config
from vllm.transformers_utils.model_arch_config_convertor import (
DeepseekV4ModelArchConfigConvertor,
+1 -3
View File
@@ -9,12 +9,10 @@ from vllm.v1.engine.async_llm import AsyncLLM
def test_mp_reducer():
"""
Test that _reduce_config reducer is registered when AsyncLLM is instantiated
"""Test that _reduce_config reducer is registered when AsyncLLM is instantiated
without transformers_modules. This is a regression test for
https://github.com/vllm-project/vllm/pull/18640.
"""
# Ensure transformers_modules is not in sys.modules
if "transformers_modules" in sys.modules:
del sys.modules["transformers_modules"]
+10 -5
View File
@@ -42,7 +42,8 @@ def test_mm_encoder_attn_backend_hash_updates():
def test_language_model_only_does_not_affect_mm_hash():
"""language_model_only does not affect the ViT computation graph,
so it should not change the multimodal config hash."""
so it should not change the multimodal config hash.
"""
base_hash = MultiModalConfig().compute_hash()
lm_only_hash = MultiModalConfig(language_model_only=True).compute_hash()
assert base_hash == lm_only_hash
@@ -50,7 +51,8 @@ def test_language_model_only_does_not_affect_mm_hash():
def test_language_model_only_affects_model_hash():
"""language_model_only affects the LM computation graph,
so it should change the model config hash."""
so it should change the model config hash.
"""
model = "llava-hf/llava-1.5-7b-hf"
base_hash = ModelConfig(model).compute_hash()
lm_only_hash = ModelConfig(model, language_model_only=True).compute_hash()
@@ -386,7 +388,8 @@ def _resolve_mm_video_decode_device(
torchcodec_available: bool = True,
) -> dict:
"""Run processor-device then video-decode-device resolution and report
the resulting video media IO kwargs."""
the resulting video media IO kwargs.
"""
mm_config = MultiModalConfig(
mm_processor_kwargs={} if device is None else {"device": device},
mm_tensor_ipc=mm_tensor_ipc, # type: ignore[arg-type]
@@ -416,7 +419,8 @@ def _resolve_mm_video_decode_device(
def test_auto_video_decode_uses_nvdec_on_encoder_instance():
"""The processor runs on the accelerator there, so decoded frames should
stay on-device too."""
stay on-device too.
"""
assert _resolve_mm_video_decode_device(ec_role="ec_producer") == {
"backend": "torchcodec",
"device": "cuda",
@@ -428,7 +432,8 @@ def test_auto_video_decode_stays_on_cpu_off_encoder_instance(
ec_role: ECRole | None,
):
"""The processor stays on CPU off encode-only instances, so video
decoding should too."""
decoding should too.
"""
assert _resolve_mm_video_decode_device(ec_role=ec_role) == {}
@@ -34,7 +34,8 @@ def _make_hf_config(**kwargs) -> PretrainedConfig:
@pytest.mark.cpu_test
def test_dict_overrides_are_not_forwarded_to_draft():
"""Dict overrides are target-specific key patches; the draft must get
only the architecture-mapping override."""
only the architecture-mapping override.
"""
composed = SpeculativeConfig.compose_draft_hf_overrides(
{"max_position_embeddings": 1234}
)
@@ -50,7 +51,8 @@ def test_none_overrides_fall_back_to_arch_mapping():
@pytest.mark.cpu_test
def test_callable_overrides_reach_the_draft_config():
"""A callable override (config-to-config transform) composes with the
architecture-mapping override and is applied to the draft config."""
architecture-mapping override and is applied to the draft config.
"""
def shrink(hf_config: PretrainedConfig) -> PretrainedConfig:
hf_config.num_hidden_layers = 1
@@ -67,7 +69,8 @@ def test_callable_overrides_reach_the_draft_config():
@pytest.mark.cpu_test
def test_arch_mapping_applies_before_callable_override():
"""The static arch-mapping override runs first, so the user callable
observes (and may adjust) the post-mapping config."""
observes (and may adjust) the post-mapping config.
"""
seen_architectures: list[str] = []
def record(hf_config: PretrainedConfig) -> PretrainedConfig:
@@ -128,7 +131,8 @@ def test_composed_override_is_picklable():
the composed override must be picklable. A nested local closure is not
(it raised ``Can't get local object`` on DFlashDraftModel); a
``functools.partial`` over a module-referenceable static method is.
Guard against regressing to a closure."""
Guard against regressing to a closure.
"""
composed = SpeculativeConfig.compose_draft_hf_overrides(_module_level_shrink)
assert isinstance(composed, functools.partial)
@@ -99,7 +99,8 @@ def test_independent_draft_model_keeps_its_own_limit(
vllm_caplog: pytest.LogCaptureFixture,
):
"""An independent AR draft may genuinely have a smaller context than the
target; its max_position_embeddings must not be resized."""
target; its max_position_embeddings must not be resized.
"""
target_model_config = ModelConfig(
AR_MODEL, hf_overrides={"max_position_embeddings": 8192}
)
+2 -2
View File
@@ -11,7 +11,7 @@ from vllm.platforms import current_platform
def check_cuda_context():
"""Check CUDA driver context status"""
"""Check CUDA driver context status."""
try:
cuda = ctypes.CDLL("libcuda.so")
device = ctypes.c_int()
@@ -22,7 +22,7 @@ def check_cuda_context():
def run_cuda_test_in_thread(device_input, expected_device_id):
"""Run CUDA context test in separate thread for isolation"""
"""Run CUDA context test in separate thread for isolation."""
try:
# New thread should have no CUDA context initially
valid_before, device_before = check_cuda_context()
+1 -2
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Test that platform imports do not prematurely initialize CUDA.
"""Test that platform imports do not prematurely initialize CUDA.
This is critical for Ray-based multi-GPU setups where workers need to
set CUDA_VISIBLE_DEVICES after importing vLLM but before CUDA is initialized.
@@ -44,8 +44,7 @@ def _make_request(stop, include_stop_str_in_output: bool, min_tokens: int = 0):
def test_stop_string_while_stop_token_terminates(include_stop_str_in_output: bool):
"""
This test verifies that the detokenizer correctly handles the case where
"""This test verifies that the detokenizer correctly handles the case where
the generated token sequence contains both:
- a stop token
- an <eos> token
@@ -67,7 +66,6 @@ def test_stop_string_while_stop_token_terminates(include_stop_str_in_output: boo
where stop-string evaluation occurs before the early-return on
stop_terminated.
"""
# Generate text "abcdeZ" and tokenize it.
generated_text = "abcde"
eos_token = "Z"
+11 -10
View File
@@ -18,13 +18,13 @@ DP_RANK = 0
@pytest.fixture
def random_port():
"""Generate a random port number for testing"""
"""Generate a random port number for testing."""
return random.randint(10000, 59900)
@pytest.fixture
def publisher_config(random_port, request):
"""Create a publisher config with inproc transport"""
"""Create a publisher config with inproc transport."""
how = request.param if hasattr(request, "param") else "inproc"
if how == "inproc":
@@ -47,7 +47,7 @@ def publisher_config(random_port, request):
@pytest.fixture
def publisher(publisher_config):
"""Create and return a publisher instance"""
"""Create and return a publisher instance."""
pub = EventPublisherFactory.create(publisher_config, DP_RANK)
yield pub
pub.shutdown()
@@ -55,7 +55,7 @@ def publisher(publisher_config):
@pytest.fixture
def subscriber(publisher_config):
"""Create and return a subscriber for testing"""
"""Create and return a subscriber for testing."""
endpoint = publisher_config.endpoint
replay_endpoint = publisher_config.replay_endpoint
@@ -74,7 +74,7 @@ def subscriber(publisher_config):
class MockSubscriber:
"""Helper class to receive and verify published events"""
"""Helper class to receive and verify published events."""
def __init__(
self,
@@ -113,7 +113,7 @@ class MockSubscriber:
self.decoder = msgspec.msgpack.Decoder(type=decode_type)
def receive_one(self, timeout=1000) -> tuple[int, SampleBatch] | None:
"""Receive a single message with timeout"""
"""Receive a single message with timeout."""
if not self.sub.poll(timeout):
return None
@@ -127,7 +127,7 @@ class MockSubscriber:
return seq, data
def request_replay(self, start_seq: int, socket_idx: int = 0) -> None:
"""Request replay of messages starting from start_seq"""
"""Request replay of messages starting from start_seq."""
if not self.replay_sockets:
raise ValueError("Replay sockets not initialized")
if socket_idx >= len(self.replay_sockets):
@@ -138,7 +138,7 @@ class MockSubscriber:
)
def receive_replay(self, socket_idx: int = 0) -> list[tuple[int, SampleBatch]]:
"""Receive replayed messages from a specific replay socket"""
"""Receive replayed messages from a specific replay socket."""
if not self.replay_sockets:
raise ValueError("Replay sockets not initialized")
if socket_idx >= len(self.replay_sockets):
@@ -170,7 +170,7 @@ class MockSubscriber:
return replayed
def close(self):
"""Clean up resources"""
"""Clean up resources."""
self.sub.close()
for replay in self.replay_sockets:
replay.close()
@@ -179,7 +179,8 @@ class MockSubscriber:
@pytest.fixture
def enable_ray_v2_backend():
"""Set env vars for the Ray V2 executor backend and shut down Ray
between tests."""
between tests.
"""
import ray
saved = {
+5 -6
View File
@@ -1,11 +1,10 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
WARNING: This test runs in both single-node (4 GPUs) and multi-node
(2 node with 2 GPUs each) modes. If the test only uses 2 GPUs, it is
important to set the distributed backend to "mp" to avoid Ray scheduling
all workers in a node other than the head node, which can cause the test
to fail.
"""WARNING: This test runs in both single-node (4 GPUs) and multi-node
(2 node with 2 GPUs each) modes. If the test only uses 2 GPUs, it is
important to set the distributed backend to "mp" to avoid Ray scheduling
all workers in a node other than the head node, which can cause the test
to fail.
"""
import json
+11 -13
View File
@@ -29,7 +29,7 @@ def test_eplb_load_stats_reduce_across_ranks():
def test_basic_rebalance():
"""Test basic rebalancing functionality"""
"""Test basic rebalancing functionality."""
# Example from https://github.com/deepseek-ai/eplb
weight = torch.tensor(
[
@@ -92,7 +92,7 @@ def test_basic_rebalance():
def test_single_gpu_case():
"""Test single GPU case"""
"""Test single GPU case."""
weight = torch.tensor([[10, 20, 30, 40]])
num_replicas = 4
num_groups = 1
@@ -115,7 +115,7 @@ def test_single_gpu_case():
def test_equal_weights():
"""Test case with equal weights"""
"""Test case with equal weights."""
weight = torch.tensor([[50, 50, 50, 50, 50, 50, 50, 50]])
num_replicas = 8
num_groups = 2
@@ -139,7 +139,7 @@ def test_equal_weights():
def test_extreme_weight_imbalance():
"""Test extreme weight imbalance case"""
"""Test extreme weight imbalance case."""
weight = torch.tensor([[1000, 1, 1, 1, 1, 1, 1, 1]])
num_replicas = 12
num_groups = 2
@@ -162,7 +162,7 @@ def test_extreme_weight_imbalance():
def test_multiple_layers():
"""Test multiple layers case"""
"""Test multiple layers case."""
weight = torch.tensor(
[
[10, 20, 30, 40, 50, 60], # First layer
@@ -195,7 +195,7 @@ def test_multiple_layers():
def test_parameter_validation():
"""Test parameter validation"""
"""Test parameter validation."""
weight = torch.tensor([[10, 20, 30, 40]])
# Test non-divisible case - this should handle normally without throwing
@@ -213,7 +213,7 @@ def test_parameter_validation():
def test_small_scale_hierarchical():
"""Test small-scale hierarchical load balancing"""
"""Test small-scale hierarchical load balancing."""
weight = torch.tensor(
[
[100, 50, 200, 75, 150, 25, 300, 80], # 8 experts
@@ -243,7 +243,7 @@ def test_small_scale_hierarchical():
def test_global_load_balance_fallback():
"""Test global load balancing fallback case"""
"""Test global load balancing fallback case."""
# When num_groups % num_nodes != 0, should fall back to global load
# balancing
weight = torch.tensor([[10, 20, 30, 40, 50, 60]])
@@ -265,7 +265,7 @@ def test_global_load_balance_fallback():
@pytest.mark.parametrize("device", ["cpu", "cuda"])
def test_device_compatibility(device):
"""Test device compatibility"""
"""Test device compatibility."""
if device == "cuda" and not torch.cuda.is_available():
pytest.skip("CUDA not available")
@@ -287,8 +287,7 @@ def test_device_compatibility(device):
def test_additional_cases():
"""Test more edge cases and different parameter combinations"""
"""Test more edge cases and different parameter combinations."""
# Test case 1: Large-scale distributed setup
weight1 = torch.tensor(
[[50, 100, 75, 120, 90, 60, 80, 110, 40, 70, 95, 85, 65, 55, 45, 35]]
@@ -320,8 +319,7 @@ def test_additional_cases():
def test_compute_logical_maps_with_negative_indices():
"""
Test that compute_logical_maps correctly handles physical slots containing
"""Test that compute_logical_maps correctly handles physical slots containing
-1 (unused slots).
"""
# 2 layers, 6 physical slots, 4 logical experts.
+2 -3
View File
@@ -57,9 +57,8 @@ def test_reuse_across_multiple_cycles():
def test_producer_consumer():
"""
This test uses the CpuGpuEvent to synchronize reads and writes to/from a shared GPU
tensor on multiple CPU threads.
"""This test uses the CpuGpuEvent to synchronize reads and writes to/from a shared
GPU tensor on multiple CPU threads.
"""
worker_stream = torch.cuda.Stream()
# Create a single element counter that will be shared between two threads
+6 -13
View File
@@ -56,8 +56,7 @@ def create_expert_indices_with_redundancy(
total_physical_experts: int,
redundancy_config: list[int], # redundancy for each logical expert
) -> torch.Tensor:
"""
Create expert indices with redundancy.
"""Create expert indices with redundancy.
Args:
num_layers: number of layers
@@ -67,6 +66,7 @@ def create_expert_indices_with_redundancy(
Returns:
indices: Shape (num_layers, total_physical_experts)
"""
assert sum(redundancy_config) == total_physical_experts
assert len(redundancy_config) == num_logical_experts
@@ -95,8 +95,7 @@ def create_expert_weights(
device: torch.device,
physical_to_logical_mapping: torch.Tensor,
) -> list[list[torch.Tensor]]:
"""
Create fake expert weights tensor for testing.
"""Create fake expert weights tensor for testing.
Use `arange` to generate predictable weights values, based on logical
expert ID.
@@ -105,6 +104,7 @@ def create_expert_weights(
Args:
physical_to_logical_mapping: Shape (num_layers, num_local_experts)
mapping[layer, physical_pos] = logical_expert_id
"""
expert_weights = []
@@ -209,9 +209,7 @@ def verify_redundant_experts_have_same_weights(
world_size: int,
num_local_experts: int,
) -> bool:
"""
Verify that all replicas of the same logical expert have the same weights.
"""
"""Verify that all replicas of the same logical expert have the same weights."""
num_layers = len(expert_weights)
total_physical_experts = world_size * num_local_experts
@@ -568,7 +566,6 @@ def test_rearrange_expert_weights_with_redundancy(
eplb_communicator,
):
"""Test the functionality of rearranging expert weights with redundancy."""
if eplb_communicator == "nixl" and not has_nixl():
pytest.skip("NIXL is not available")
if torch.accelerator.device_count() < world_size:
@@ -678,7 +675,6 @@ def test_async_transfer_layer_without_mtp(
eplb_communicator: str,
):
"""Exercise async EPLB transfer path without MTP/spec decode."""
if eplb_communicator == "nixl" and not has_nixl():
pytest.skip("NIXL is not available")
if torch.accelerator.device_count() < world_size:
@@ -696,11 +692,9 @@ def test_async_transfer_layer_without_mtp(
@pytest.mark.parametrize("world_size", [2, 4])
def test_rearrange_expert_weights_no_change(world_size):
"""
Test that when the indices do not change, the weights should remain
"""Test that when the indices do not change, the weights should remain
unchanged.
"""
if torch.accelerator.device_count() < world_size:
pytest.skip(f"Need at least {world_size} GPUs to run the test")
distributed_run(
@@ -800,7 +794,6 @@ def _test_rearrange_expert_weights_profile_mode(env, world_size) -> None:
@pytest.mark.parametrize("world_size", [2, 4])
def test_rearrange_expert_weights_profile_mode(world_size):
"""Test profile mode (should not copy actual weights)"""
if torch.accelerator.device_count() < world_size:
pytest.skip(f"Need at least {world_size} GPUs to run the test")
distributed_run(
@@ -124,7 +124,8 @@ def _build_processed_layer(
backend: str, raw: dict[str, torch.Tensor], device: torch.device
) -> tuple[CompressedTensorsW4A4Nvfp4MoEMethod, _RoutedExpertsStub]:
"""Create the method + layer, load ``raw`` and run the real
process_weights_after_loading (which builds the flashinfer kernel)."""
process_weights_after_loading (which builds the flashinfer kernel).
"""
moe_config = _make_moe_config(backend)
method = CompressedTensorsW4A4Nvfp4MoEMethod(moe_config, "layer.0", use_a16=False)
layer = _RoutedExpertsStub(moe_config).to(device)
@@ -157,7 +158,8 @@ def _assert_tensors_equal(
def _simulate_eplb_rearrangement(layer: torch.nn.Module, perm: torch.Tensor) -> None:
"""Permute experts of every registered Parameter in place, the way
EPLB's in-place rearrangement moves expert slices along dim 0."""
EPLB's in-place rearrangement moves expert slices along dim 0.
"""
with torch.no_grad():
for _, param in layer.named_parameters():
param.copy_(param[perm])
@@ -165,7 +167,8 @@ def _simulate_eplb_rearrangement(layer: torch.nn.Module, perm: torch.Tensor) ->
def _ensure_world1_distributed() -> None:
"""The enable_eplb config makes weight processing all-reduce activation
scale amaxes over the EP group, so a (single-rank) EP group must exist."""
scale amaxes over the EP group, so a (single-rank) EP group must exist.
"""
from vllm.distributed.parallel_state import (
ensure_model_parallel_initialized,
init_distributed_environment,
@@ -253,7 +256,8 @@ def test_nvfp4_eplb_rearrangement_matches_reload(backend: str) -> None:
def test_fp8_per_tensor_alphas_registered_and_aliased() -> None:
"""CPU-only: the fp8 per-tensor oracle must register the fused
(w_scale * a_scale) products as layer Parameters aliased by the quant
config, so EPLB rearrangement keeps kernels consistent."""
config, so EPLB rearrangement keeps kernels consistent.
"""
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
Fp8MoeBackend,
make_fp8_moe_quant_config,
+2 -6
View File
@@ -75,8 +75,7 @@ def test_eplb_spec_decode(
monkeypatch: pytest.MonkeyPatch,
model_setup: tuple[str, str, str, int, float],
):
"""
Test the correctness of EPLB speculative decoding with GSM8K dataset.
"""Test the correctness of EPLB speculative decoding with GSM8K dataset.
Applicable to MoE models with mtp or eagle spec decode.
"""
method, model_name, spec_model_name, tp_size, expected_gsm8k_value = model_setup
@@ -109,10 +108,7 @@ def test_eplb_spec_decode(
@large_gpu_mark(min_gb=80)
def test_eplb_spec_decode_qwen3_next_mtp_async() -> None:
"""
Ensure async EPLB works with MTP speculative decoding for Qwen3-Next.
"""
"""Ensure async EPLB works with MTP speculative decoding for Qwen3-Next."""
TASK = "gsm8k"
FILTER = "exact_match,strict-match"
RTOL = 0.03
+2 -4
View File
@@ -30,8 +30,7 @@ def _make_model_state(
def test_commit_eplb_maps_shape_change():
"""
When the number of physical experts changes, resize the active map within its
"""When the number of physical experts changes, resize the active map within its
preallocated storage.
"""
num_layers, num_logical, num_physical = 2, 4, 6
@@ -65,8 +64,7 @@ def test_commit_eplb_maps_shape_change():
def test_commit_eplb_maps_for_layer_logical_padding():
"""
Test that logical_to_physical_map is padded with -1 to fill the
"""Test that logical_to_physical_map is padded with -1 to fill the
pre-allocated slots when the new map has fewer replicas than the max.
"""
num_layers, num_logical, num_physical = 2, 4, 6
+13 -17
View File
@@ -20,27 +20,26 @@ class EventSample(
tag=True, # type: ignore
array_like=True, # type: ignore
):
"""Test event for publisher testing"""
"""Test event for publisher testing."""
id: int
value: str
class SampleBatch(EventBatch):
"""Test event batch for publisher testing"""
"""Test event batch for publisher testing."""
events: list[EventSample]
def create_test_events(count: int) -> SampleBatch:
"""Create a batch of test events"""
"""Create a batch of test events."""
events = [EventSample(id=i, value=f"test-{i}") for i in range(count)]
return SampleBatch(ts=time.time(), events=events)
def test_basic_publishing(publisher, subscriber):
"""Test basic event publishing works"""
"""Test basic event publishing works."""
test_batch = create_test_events(5)
publisher.publish(test_batch)
@@ -58,7 +57,7 @@ def test_basic_publishing(publisher, subscriber):
def test_multiple_events(publisher, subscriber):
"""Test publishing and receiving multiple event batches"""
"""Test publishing and receiving multiple event batches."""
for _ in range(10):
batch = create_test_events(2)
publisher.publish(batch)
@@ -75,7 +74,7 @@ def test_multiple_events(publisher, subscriber):
def test_replay_mechanism(publisher, subscriber):
"""Test the replay mechanism works correctly"""
"""Test the replay mechanism works correctly."""
for _ in range(19):
batch = create_test_events(1)
publisher.publish(batch)
@@ -96,7 +95,7 @@ def test_replay_mechanism(publisher, subscriber):
def test_replay_includes_topic(publisher, subscriber, publisher_config):
"""Test that replay responses include the topic, matching PUB format"""
"""Test that replay responses include the topic, matching PUB format."""
for _ in range(5):
publisher.publish(create_test_events(1))
@@ -115,7 +114,7 @@ def test_replay_includes_topic(publisher, subscriber, publisher_config):
def test_buffer_limit(publisher, subscriber, publisher_config):
"""Test buffer limit behavior"""
"""Test buffer limit behavior."""
buffer_size = publisher_config.buffer_steps
# Publish more events than the buffer can hold
@@ -139,9 +138,7 @@ def test_buffer_limit(publisher, subscriber, publisher_config):
def test_topic_filtering(publisher_config):
"""
Test that a subscriber only receives messages matching its topic filter
"""
"""Test that a subscriber only receives messages matching its topic filter."""
publisher_config.replay_endpoint = None
publisher_config.topic = "foo"
@@ -174,7 +171,7 @@ def test_topic_filtering(publisher_config):
def test_high_volume(publisher, subscriber):
"""Test publishing and receiving a high volume of events"""
"""Test publishing and receiving a high volume of events."""
num_batches = 10_000
events_per_batch = 100
@@ -210,7 +207,7 @@ def test_high_volume(publisher, subscriber):
def test_null_publisher():
"""Test that NullEventPublisher can be used without errors"""
"""Test that NullEventPublisher can be used without errors."""
publisher = NullEventPublisher(DP_RANK)
# This should not raise any errors
@@ -220,8 +217,7 @@ def test_null_publisher():
def test_data_parallel_rank_tagging(publisher_config):
"""Test that events are properly tagged with their data parallel rank"""
"""Test that events are properly tagged with their data parallel rank."""
publisher_config.topic = "foo"
pub_0 = EventPublisherFactory.create(publisher_config, DP_RANK)
pub_1 = EventPublisherFactory.create(publisher_config, DP_RANK + 1)
@@ -285,7 +281,7 @@ def test_data_parallel_rank_tagging(publisher_config):
def test_event_publisher_factory(random_port):
"""Test event publisher factory creation behavior under different configurations"""
"""Test event publisher factory creation behavior under different configurations."""
from vllm.config.kv_events import KVEventsConfig
from vllm.distributed.kv_events import ZmqEventPublisher
@@ -49,7 +49,6 @@ def verify_round_robin_pattern(expert_map, ep_rank, ep_size, global_num_experts)
@pytest.mark.parametrize("world_size", [2, 4])
def test_expert_placement_various_sizes(expert_placement_strategy, world_size):
"""Test round_robin expert placement with various expert counts."""
# Test with different global_num_experts values
# Include both divisible and non-divisible cases
if world_size == 2:
@@ -116,7 +115,6 @@ def test_expert_placement_various_sizes(expert_placement_strategy, world_size):
@pytest.mark.parametrize("world_size", [2, 4])
def test_expert_placement_edge_cases(expert_placement_strategy, world_size):
"""Test edge cases for round_robin expert placement."""
# Test case 1: ep_size = 1 (should return None for expert_map)
local_num_experts, expert_map, _ = determine_expert_map(
ep_size=1,
@@ -139,7 +137,6 @@ def test_expert_placement_edge_cases(expert_placement_strategy, world_size):
def test_determine_expert_map_comprehensive():
"""Test of determine_expert_map function with various configurations."""
# Test cases: (ep_size, ep_rank, global_num_experts,
# expert_placement_strategy, expected_local, expected_map_pattern)
test_cases = [
+7 -4
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Tests for MNNVL AllToAll operations.
"""Tests for MNNVL AllToAll operations.
Requires: docker run ... --cap-add=SYS_PTRACE ...
Run: pytest tests/distributed/test_mnnvl_alltoall.py -v
@@ -87,6 +86,7 @@ def _run_worker(rank, world_size, port, worker_fn, dp_size, dp_port, err_queue):
Otherwise use tp=world_size (default for EP-based tests).
dp_port: Separate port for the DP master (only used when dp_size is set).
err_queue: Queue for propagating tracebacks to the parent process.
"""
try:
os.environ.pop("CUDA_VISIBLE_DEVICES", None)
@@ -115,6 +115,7 @@ def _init_dp_environment(world_size, rank, port, dp_size, dp_port):
Args:
port: Port for torch.distributed init.
dp_port: Separate port for the DP master group init.
"""
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.config.parallel import ParallelConfig
@@ -159,7 +160,8 @@ def _make_forward_context(rank, world_size, num_tokens_per_rank):
class _AttnMeta:
"""Minimal placeholder so set_forward_context's
``attn_metadata is not None`` guard (forward_context.py:334)
is satisfied. The real DPMetadata is built from num_tokens_across_dp."""
is satisfied. The real DPMetadata is built from num_tokens_across_dp.
"""
dp_metadata = None
@@ -458,7 +460,8 @@ def _one_sided_workspace_grow_worker(rank, world_size):
def test_one_sided_manager_workspace_grow(world_size):
"""A later initialize() with a larger per-token payload must grow the
workspace and rebuild MoeAlltoAll; a later initialize() with a smaller
payload must no-op."""
payload must no-op.
"""
_spawn_workers(
_one_sided_workspace_grow_worker,
world_size,
+7 -5
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Test that MessageQueue uses the local node's IP for binding,
"""Test that MessageQueue uses the local node's IP for binding,
not a remote master_addr. This validates the fix for cross-node
data-parallel where each DP group leader must bind to its own IP.
@@ -22,7 +21,8 @@ from vllm.utils.network_utils import get_ip
def test_mq_bind_with_local_ip():
"""MessageQueue with remote readers should successfully bind
when connect_ip is the local node's IP."""
when connect_ip is the local node's IP.
"""
# n_reader=2, n_local_reader=1 means 1 remote reader,
# which triggers the remote ZMQ socket bind.
mq = MessageQueue(
@@ -44,7 +44,8 @@ def test_mq_bind_with_local_ip():
def test_mq_bind_with_non_local_ip_fails():
"""MessageQueue should fail to bind when connect_ip is a
non-local IP address (simulating the bug where master_addr
from a different node was used)."""
from a different node was used).
"""
# Use a non-local IP that we definitely can't bind to.
# 198.51.100.1 is from TEST-NET-2 (RFC 5737), never locally assigned.
non_local_ip = "198.51.100.1"
@@ -58,7 +59,8 @@ def test_mq_bind_with_non_local_ip_fails():
def test_mq_bind_defaults_to_local_ip():
"""When connect_ip is None, MessageQueue should auto-detect
the local IP and bind successfully."""
the local IP and bind successfully.
"""
mq = MessageQueue(
n_reader=2,
n_local_reader=1,
+2 -4
View File
@@ -1,8 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Integration tests for MultiprocExecutor at the executor level.
"""Integration tests for MultiprocExecutor at the executor level.
This test directly tests the executor without going through the LLM interface,
focusing on executor initialization, RPC calls, and distributed execution.
"""
@@ -377,8 +376,7 @@ def test_multiproc_executor_properties():
@multi_gpu_test(num_gpus=4)
def test_multiproc_executor_multi_node():
"""
Test MultiprocExecutor with multi-node configuration.
"""Test MultiprocExecutor with multi-node configuration.
This simulates 2 nodes with TP=4:
- Node 0 (rank 0): Uses GPUs 0,1 (CUDA_VISIBLE_DEVICES=0,1) with TP=2
- Node 1 (rank 1): Uses GPUs 2,3 (CUDA_VISIBLE_DEVICES=2,3) with TP=2
+5 -6
View File
@@ -1,11 +1,10 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
WARNING: This test runs in both single-node (4 GPUs) and multi-node
(2 node with 2 GPUs each) modes. If the test only uses 2 GPUs, it is
important to set the distributed backend to "mp" to avoid Ray scheduling
all workers in a node other than the head node, which can cause the test
to fail.
"""WARNING: This test runs in both single-node (4 GPUs) and multi-node
(2 node with 2 GPUs each) modes. If the test only uses 2 GPUs, it is
important to set the distributed backend to "mp" to avoid Ray scheduling
all workers in a node other than the head node, which can cause the test
to fail.
"""
import json
+2 -1
View File
@@ -495,7 +495,8 @@ def test_pynccl_suspend_resume_idempotent():
def test_pynccl_suspend_noop_without_symbol():
"""RCCL / NCCL < 2.29.7 lack ncclCommSuspend; suspend() must no-op, not
crash, and resume() must then no-op too since nothing was suspended."""
crash, and resume() must then no-op too since nothing was suspended.
"""
from unittest.mock import Mock
comm = object.__new__(PyNcclCommunicator)
+1 -2
View File
@@ -396,8 +396,7 @@ def test_custom_quick_allreduce_bf16_cast(monkeypatch: pytest.MonkeyPatch):
def qr_variable_input(rank, world_size):
"""
When the tensor parallelism is set to 4 or 8, frequent changes
"""When the tensor parallelism is set to 4 or 8, frequent changes
in the input shape can cause QuickReduce to hang (this issue
has been observed with the gpt_oss model).
"""
+1 -3
View File
@@ -1,8 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Integration tests for RayExecutorV2 at the executor level.
"""Integration tests for RayExecutorV2 at the executor level.
Validates executor initialization, placement group support, RPC calls,
and distributed execution with various TP/PP configurations.
"""
@@ -299,7 +298,6 @@ def test_ray_v2_run_refs_stored_for_monitoring(executor):
@pytest.mark.parametrize("tp_size, pp_size", [(2, 1), (2, 2)])
def test_ray_v2_single_node_generation(tp_size, pp_size):
"""End-to-end LLM generation with RayExecutorV2."""
llm = LLM(
model=MODEL,
tensor_parallel_size=tp_size,
@@ -1,8 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Orchestration-level integration tests for RayExecutorV2.
"""
"""Orchestration-level integration tests for RayExecutorV2."""
import gc
import os
@@ -25,7 +23,8 @@ def _ray_init():
Without this, workers cannot unpickle actor classes defined in the
``tests`` package, causing FunctionActorManager to fall back to
TemporaryActor which drops async method signatures."""
TemporaryActor which drops async method signatures.
"""
project_root = str(pathlib.Path(__file__).resolve().parents[2])
ray.init(
ignore_reinit_error=True,
@@ -158,8 +157,7 @@ def test_multi_replicas_with_bundle_indices(ray_init):
def test_env_var_and_runtime_env_propagation():
"""
Verify env vars (NCCL_, HF_) and parallel_config.ray_runtime_env
"""Verify env vars (NCCL_, HF_) and parallel_config.ray_runtime_env
propagate to RayWorkerProc actors.
"""
sentinel_vars = {
+125 -64
View File
@@ -53,7 +53,8 @@ META = torch.device("meta")
class _FakeLayer:
"""Stand-in for a leaf module. The planner only stores it and keys on
``id()``; nothing in the plan path touches its attributes."""
``id()``; nothing in the plan path touches its attributes.
"""
def __init__(self, tag: str) -> None:
self.tag = tag
@@ -64,7 +65,8 @@ class _FakeLayer:
def _module(layer, copies):
"""What the bake records for one leaf module: a scatter list whose entries
all carry that module as their ``layer``."""
all carry that module as their ``layer``.
"""
for c in copies:
c.layer = layer
return copies
@@ -81,7 +83,8 @@ def _copy(
dtype=torch.bfloat16,
):
"""A `_Scatter` as the bake would record it: source key, owning layer, and
the meta destination region."""
the meta destination region.
"""
stride = (
(1,)
if len(shape) == 1
@@ -140,7 +143,8 @@ def _planner(baked, *, name_meta, live=None, held_by=None, group_lens=None):
def _one_module_per_layer(n_layers, *, dtype="bfloat16", numel=4):
"""`(baked, name_meta, names, group_lens)` for a pre / N-layer / post model
where every gather group is one fully-baked leaf module."""
where every gather group is one fully-baked leaf module.
"""
names = ["embed.weight"]
names += [f"model.layers.{i}.w" for i in range(n_layers)]
names += ["norm.weight"]
@@ -161,7 +165,8 @@ def _one_module_per_layer(n_layers, *, dtype="bfloat16", numel=4):
class TestFakeOpChains:
"""Every allowlisted op must append itself to the chain and hand back a
child whose shape/dtype PyTorch itself computed. The chain is the wire
format the producer replays, so its exact contents are load-bearing."""
format the producer replays, so its exact contents are load-bearing.
"""
def _fake(self, shape=(4, 6), dtype=torch.bfloat16, sink=None):
return FakeRDTTensor(
@@ -222,7 +227,8 @@ class TestFakeOpChains:
@pytest.mark.parametrize("op,n", [("chunk", 2), ("unbind", 4)])
def test_multi_return_op_emits_one_child_per_output(self, op, n):
"""chunk/unbind hand back a tuple; each child carries the base op plus a
trailing __getitem__(i) so the producer can index the replayed result."""
trailing __getitem__(i) so the producer can index the replayed result.
"""
parts = getattr(self._fake(), op)(*((n,) if op == "chunk" else ()), 0)
assert isinstance(parts, tuple)
assert len(parts) == n
@@ -238,7 +244,8 @@ class TestFakeOpChains:
class TestFakeUnsupportedOps:
"""Anything that needs real data must fail loudly at bake time rather than
silently transferring the wrong bytes."""
silently transferring the wrong bytes.
"""
def _fake(self):
return FakeRDTTensor(
@@ -278,7 +285,8 @@ class TestFakeUnsupportedOps:
class TestBakeRecording:
"""During the dry run the fake's ``copy_`` is the data sink: it records the
source chain plus the meta destination's strided region and moves nothing."""
source chain plus the meta destination's strided region and moves nothing.
"""
def _recorder_and_fake(self, shape=(4, 6)):
rec = BakeSink()
@@ -314,7 +322,8 @@ class TestBakeRecording:
def test_unattributed_copy_is_live_but_unrecorded(self):
"""A copy_ with no loader stamp cannot be attributed to a param, so its
module must fall back to the plain load — but the name still moved data."""
module must fall back to the plain load — but the name still moved data.
"""
rec, fake = self._recorder_and_fake()
rec.current = None
torch.empty((4, 6), dtype=torch.bfloat16, device=META).copy_(fake)
@@ -354,7 +363,8 @@ class TestBakeRecording:
producer packs what the replay yields, so the record must carry the
POST-chain dtype: taking it from the source name's metadata instead sizes
the slice with the wrong itemsize and shifts every later slice in the
chunk, carving the packed blob differently on the two sides."""
chunk, carving the packed blob differently on the two sides.
"""
rec, fake = self._recorder_and_fake(shape=(4,))
layer = _FakeLayer("reinterpreted")
rec.current = (layer, "weight")
@@ -374,7 +384,8 @@ class TestBakeRecording:
the producer packs what the chain yields, and nothing downstream can
catch a mismatch — the consumer's buffer view is exactly prod(dest.shape)
elements, so it reshapes cleanly over bytes laid out at other offsets.
So the bake refuses rather than recording a slice it cannot carve."""
So the bake refuses rather than recording a slice it cannot carve.
"""
rec, fake = self._recorder_and_fake(shape=(1, 6))
rec.current = (_FakeLayer("broadcast"), "weight")
param = torch.empty((4, 6), dtype=torch.bfloat16, device=META)
@@ -390,7 +401,8 @@ class TestBakeRecording:
class TestConsumerIdentity:
"""Every worker in the fleet needs a DISTINCT id in 0..C-1: it selects the
worker's producer block and keys the producer's per-consumer serve ring, so
a collision silently serves two workers out of one ring."""
a collision silently serves two workers out of one ring.
"""
def _engine(self, *, dp_index, rank, world_size):
eng = object.__new__(ShardedRDTWeightTransferEngine)
@@ -427,7 +439,8 @@ class TestConsumerIdentity:
def test_independent_engines_offset_into_distinct_ranges(self):
"""A fleet of separate engines restarts _global_worker_index at 0 in
each, so without the replica offset every engine would claim 0..w-1.
The driver sets replica_rank on the payload; these are the ids it buys."""
The driver sets replica_rank on the payload; these are the ids it buys.
"""
ids = [
self._ids(num_consumers=8, workers=range(4), replica_rank=r, num_replicas=2)
for r in (0, 1)
@@ -453,7 +466,8 @@ def _router(
group_lens=None,
):
"""A router over ``n`` single-name groups unless told otherwise, so a test
can state expectations in group indices."""
can state expectations in group indices.
"""
if names is None:
n = len(name_owner_class or []) or (len(group_lens or []) or 1)
names = [f"g{i}" for i in range(n)]
@@ -482,7 +496,8 @@ class TestRdtRouter:
def test_every_producer_nic_carries_traffic(self):
"""16 producers / 8 consumers: the block rule spreads each consumer's
pulls over its block, alternating by group, so no producer sits idle.
Plain `consumer_id % len(owners)` would idle half of them."""
Plain `consumer_id % len(owners)` would idle half of them.
"""
r = _router(16, 8, group_lens=[1] * 95)
names = [f"g{i}" for i in range(95)]
for c in range(8):
@@ -500,7 +515,8 @@ class TestRdtRouter:
def test_pipeline_stages_route_to_the_owning_stage(self):
"""2 stages x 8 ranks: groups 0-2 on stage 0, 3-5 on stage 1. Two owner
sets, one per stage; each consumer reaches both."""
sets, one per stage; each consumer reaches both.
"""
r = _router(
16,
8,
@@ -516,7 +532,8 @@ class TestRdtRouter:
def test_expert_names_route_to_the_holding_ranks(self):
"""A name held by only some ranks routes to those; a replicated name
keeps the full set."""
keeps the full set.
"""
r = RdtRouter(
8,
4,
@@ -535,7 +552,8 @@ class TestRdtRouter:
def test_consumers_spread_over_an_owner_set(self):
"""Several ranks hold a name (its TP peers); the block rule must spread
consumers across them, not funnel through one."""
consumers across them, not funnel through one.
"""
r = _router(
4, 4, owner_sets=[[0, 1]], name_owner_class=[0] * 6, group_lens=[1] * 6
)
@@ -544,7 +562,8 @@ class TestRdtRouter:
def test_a_group_frees_at_every_owner_of_any_of_its_names(self):
"""The free barrier is per group, so its fan-out is the UNION over the
group's names: a rank holding none of them is not signalled."""
group's names: a rank holding none of them is not signalled.
"""
r = RdtRouter(4, 2, [[0, 1], [2, 3]], [0, 1, 0], ["a", "b", "c"], [2, 1])
assert r.group_owners(0) == [0, 1, 2, 3]
assert r.group_owners(1) == [0, 1]
@@ -614,7 +633,8 @@ class TestReplicaOverlay:
def test_adding_a_deployment_does_not_move_the_first_one(self):
"""The strongest form: an existing consumer's routes are byte-identical
before and after another deployment is provisioned."""
before and after another deployment is provisioned.
"""
one, names = self._fleet(16, 8, 1)
two, _ = self._fleet(16, 8, 2)
for consumer_id in range(8):
@@ -624,7 +644,8 @@ class TestReplicaOverlay:
def test_one_deployment_is_the_historical_carve(self):
"""Default (no width) and width == the whole fleet must agree with the
plain block rule, so a single-deployment fleet is untouched."""
plain block rule, so a single-deployment fleet is untouched.
"""
plain = _router(16, 8, group_lens=[1] * 6)
explicit, names = self._fleet(16, 8, 1)
for consumer_id in range(8):
@@ -638,7 +659,8 @@ class TestReplicaOverlay:
def test_the_overlay_holds_for_partially_owned_names(self):
"""Pipeline stages and experts too: two owner sets, one narrow. The
narrow one already forced agreement (one owner, no block to carve); the
wide one is what the carve decides."""
wide one is what the carve decides.
"""
r, names = self._fleet(
8,
4,
@@ -654,14 +676,16 @@ class TestReplicaOverlay:
def test_a_width_that_does_not_divide_the_fleet_raises(self):
"""A uniform fleet is asserted upstream in ``get_world_size``; if it ever
is not, the overlay would map two workers of one deployment onto the same
block index, so refuse rather than serve the wrong bytes."""
block index, so refuse rather than serve the wrong bytes.
"""
with pytest.raises(ValueError, match="does not divide"):
RdtRouter(4, 6, None, None, ["a"], [1], workers_per_replica=4)
def test_pulls_still_carry_the_global_consumer_id(self):
"""The carve uses the index WITHIN a deployment, but the wire must keep
the fleet-global id: it is what lets the producer tell the sharers of a
slot apart, and count their arrivals separately."""
slot apart, and count their arrivals separately.
"""
class _Method:
def __init__(self):
@@ -704,7 +728,8 @@ def _producer_pack_offsets(slices):
class TestPackedLayout:
def test_consumer_layout_matches_the_producer_rule(self):
"""Mixed dtypes, sizes that do not land on 16B — i.e. the Kimi-style
group (fp8 weights + fp32 scales + bf16 norms) that exposes alignment."""
group (fp8 weights + fp32 scales + bf16 norms) that exposes alignment.
"""
specs = [
("w.fp8", "float8_e4m3fn", [17]),
("w.scale", "float32", [3]),
@@ -779,7 +804,8 @@ class TestPackedLayout:
class TestChunkModuleScatters:
"""The chunk cut: one chunk per distinct owner class present in the copies,
ascending by class index. Derived purely from the bake plus the ownership
table, so vLLM's expert placement needs no cases."""
table, so vLLM's expert placement needs no cases.
"""
def test_unstamped_copies_form_one_chunk(self):
layer = _FakeLayer("l")
@@ -793,7 +819,8 @@ class TestChunkModuleScatters:
"""Classes are numbered by first appearance in metadata order, so the
replicated class (0) leads and the rest follow — the ordering the old
"-1 first, then ascending stamp" rule produced, now falling out of the
table instead of a special case."""
table instead of a special case.
"""
layer = _FakeLayer("moe")
names = ["norm", "e5", "e0", "e2"]
copies = [_copy(n, shape=(4,)) for n in names]
@@ -813,7 +840,8 @@ class TestChunkModuleScatters:
def test_copy_order_within_a_chunk_is_bake_order(self):
"""Scatter order within a chunk must follow the bake, not the stamp
sort: the packed layout and the producer's replay agree on keys order."""
sort: the packed layout and the producer's replay agree on keys order.
"""
layer = _FakeLayer("moe")
names = ["e3", "e1", "e2"] # one stamp, deliberately unsorted names
copies = [_copy(n, shape=(4,)) for n in names]
@@ -827,7 +855,8 @@ class TestChunkModuleScatters:
def test_stamps_interleave_across_modules_without_reordering_within(self):
"""Two modules' copies bucket by stamp; within each bucket the order is
module-then-bake order."""
module-then-bake order.
"""
a, b = _FakeLayer("a"), _FakeLayer("b")
mods = [
_module(a, [_copy("a0", shape=(4,)), _copy("a1", shape=(4,))]),
@@ -845,9 +874,10 @@ class TestChunkModuleScatters:
]
def test_scatters_carry_their_own_dtype(self):
"""dtype rides the record from the bake, where it is the fake's dtype
"""Dtype rides the record from the bake, where it is the fake's dtype
AFTER its op chain — not a plan-time lookup of the source name, which
would be wrong for any chain that reinterprets dtype."""
would be wrong for any chain that reinterprets dtype.
"""
layer = _FakeLayer("l")
eng = _planner({}, name_meta={"w": ("float32", [10])})
((_er, scatters),) = eng._chunk_module_scatters(
@@ -886,7 +916,8 @@ class TestBuildCallPlan:
def test_materialize_fires_on_a_modules_first_chunk_only(self):
"""Empty HF params are allocated once per module, by construction —
including a FusedMoE-like module whose copies span owner-class chunks."""
including a FusedMoE-like module whose copies span owner-class chunks.
"""
layer = _FakeLayer("spanning")
names = [f"w{i}" for i in range(4)]
copies = [_copy(n, shape=(4,)) for n in names]
@@ -921,7 +952,8 @@ class TestBuildCallPlan:
def test_free_signal_waits_for_a_groups_last_chunk_when_it_is_split(self):
"""The load-bearing case: with the group cut across owner-class chunks the
signal must hang off the LAST one. Signaling earlier lets the producers
drop the gather buffers while a later chunk's RDMA is still reading."""
drop the gather buffers while a later chunk's RDMA is still reading.
"""
layer = _FakeLayer("l")
names = [f"w{i}" for i in range(4)]
copies = [_copy(n, shape=(4,)) for n in names]
@@ -937,7 +969,8 @@ class TestBuildCallPlan:
def test_free_signal_timing_across_two_split_groups(self):
"""Two groups x 2 owner-class chunks each: signals land on each group's
last chunk only."""
last chunk only.
"""
names_a = [f"model.layers.0.w{i}" for i in range(4)]
names_b = [f"model.layers.1.w{i}" for i in range(4)]
baked, name_meta, stamps = {}, {}, {}
@@ -955,7 +988,8 @@ class TestBuildCallPlan:
def test_a_group_with_nothing_local_is_signaled_at_sync_start(self):
"""The owners still published it, so this consumer must signal it —
immediately, before the pipeline (signal-before-publish is tolerated),
never hung off another group's chunk."""
never hung off another group's chunk.
"""
baked, name_meta, names, group_lens = _one_module_per_layer(2)
del baked["model.layers.1.w"] # nothing baked, and not live => no pull
eng = _planner(
@@ -976,7 +1010,8 @@ class TestBuildCallPlan:
def test_modules_are_deduped_within_a_group(self):
"""Several names of one fused module map to the same scatter list; the
plan must not chunk it twice."""
plan must not chunk it twice.
"""
layer = _FakeLayer("fused")
names = ["qkv.q", "qkv.k", "qkv.v"]
copies = [_copy(n, shape=(4,)) for n in names]
@@ -992,7 +1027,8 @@ class TestBuildCallPlan:
def test_plan_is_a_pure_function_of_its_inputs(self):
"""Called twice with the same arguments it must produce the same plan —
that is what lets the engine build it once and reuse it every sync."""
that is what lets the engine build it once and reuse it every sync.
"""
baked, name_meta, names, group_lens = _one_module_per_layer(2)
eng = _planner(baked, name_meta=name_meta)
a = eng._build_call_plan(names, group_lens)
@@ -1012,7 +1048,8 @@ class TestBuildCallPlan:
class TestUnbakedNamesGuard:
"""A live name with no baked plan has no way to load — there is no fallback,
and its pull would target groups the pipeline already freed — so the plan
build fails at init, naming the names."""
build fails at init, naming the names.
"""
def test_a_live_unbaked_name_fails_the_plan_build(self):
baked, name_meta, names, group_lens = _one_module_per_layer(2)
@@ -1023,7 +1060,8 @@ class TestUnbakedNamesGuard:
def test_never_copied_names_are_dropped_entirely(self):
"""Experts owned by another EP rank no-op in their loader: not an error,
and not a chunk — their group is pre-freed if nothing else fills it."""
and not a chunk — their group is pre-freed if nothing else fills it.
"""
baked, name_meta, names, group_lens = _one_module_per_layer(2)
del baked["norm.weight"]
eng = _planner(baked, name_meta=name_meta, live=set(names) - {"norm.weight"})
@@ -1035,13 +1073,15 @@ class TestUnbakedNamesGuard:
class TestCallPlanRouting:
"""Under partial ownership each chunk must be pulled from a producer that
actually holds every name in it. Consumers bind EVERY producer (signals fan
out to all owners), so local index == trainer rank."""
out to all owners), so local index == trainer rank.
"""
def _routed_planner(
self, held_by_group, num_consumers, consumer_id, *, held_by=None
):
"""``held_by_group[g]`` = the ranks holding group g's names, so a test
can still speak in groups; ``held_by`` overrides individual names."""
can still speak in groups; ``held_by`` overrides individual names.
"""
baked, name_meta, names, group_lens = _one_module_per_layer(
len(held_by_group) - 2
)
@@ -1068,14 +1108,16 @@ class TestCallPlanRouting:
def test_two_stage_ownership_splits_chunks_between_owners(self):
"""Two PP stages, 4 groups: stage 0 holds the first half, stage 1 the
second. Every chunk goes to a holder of its names."""
second. Every chunk goes to a holder of its names.
"""
eng, names, group_lens = self._routed_planner([[0], [0], [1], [1]], 1, 0)
plan = eng._build_call_plan(names, group_lens)
assert [c.owner for c in plan.chunks] == [0, 0, 1, 1]
def test_a_multi_owner_group_still_resolves_to_one_producer(self):
"""Every chunk is served by exactly ONE producer: splitting a pull only
multiplies produce calls, since the consumer's NIC bounds it."""
multiplies produce calls, since the consumer's NIC bounds it.
"""
eng, names, group_lens = self._routed_planner([[0, 1]] * 4, 1, 0)
plan = eng._build_call_plan(names, group_lens)
assert all(0 <= c.owner < 2 for c in plan.chunks)
@@ -1083,7 +1125,8 @@ class TestCallPlanRouting:
def test_an_expert_name_routes_to_the_rank_holding_it(self):
"""PP ∩ EP: stage 0 = ranks 0-1, stage 1 = ranks 2-3, and one expert
name inside a stage-1 group is held by rank 3 alone. The name's own owner
set is what routes it."""
set is what routes it.
"""
eng, names, group_lens = self._routed_planner(
[[0, 1], [0, 1], [2, 3], [2, 3]],
1,
@@ -1098,7 +1141,8 @@ class TestCallPlanRouting:
def test_an_unheld_name_is_rejected_at_init(self):
"""A name no rank holds can never be served. The trainer raises when it
transposes the holdings; the router's validate() is the consumer-side
backstop, and either way it fails at init rather than at first pull."""
backstop, and either way it fails at init rather than at first pull.
"""
with pytest.raises(ValueError, match="empty"):
RdtRouter(2, 1, [[]], [0], ["w"], [1]).validate()
@@ -1107,14 +1151,16 @@ class TestSignalCompleteness:
"""Every gather group is signaled exactly once — after its last chunk when
the worker pulls from it, at sync start otherwise — for ANY expert
placement. A group signaled twice over-credits the barrier; a group never
signaled parks a producer credit and hangs end_sync."""
signaled parks a producer credit and hangs end_sync.
"""
def _moe_planner(self, worker_experts, *, ep_size=4, n_experts=8, n_layers=2):
"""pre / n_layers MoE layers / post. Each layer: one norm (its own
"""Pre / n_layers MoE layers / post. Each layer: one norm (its own
module, stamp -1) + n_experts expert names stamped ``e // n_local``.
The worker's bake covers only ``worker_experts`` (its placement);
foreign expert names never copied => dropped entirely, as in the real
bake."""
bake.
"""
n_local = n_experts // ep_size
names = ["embed.weight"]
group_lens = [1]
@@ -1173,8 +1219,9 @@ class TestSignalCompleteness:
assert self._signals(plan) == list(range(len(group_lens)))
def test_placement_changes_only_the_chunk_count_never_the_signals(self):
"""linear experts 0-3 hit 2 owner classes; round_robin 0,2,4,6
hits all 4. More chunks per group, identical signal set."""
"""Linear experts 0-3 hit 2 owner classes; round_robin 0,2,4,6
hits all 4. More chunks per group, identical signal set.
"""
plans = {}
for label, experts in (("linear", [0, 1, 2, 3]), ("round_robin", [0, 2, 4, 6])):
eng, names, group_lens = self._moe_planner(worker_experts=experts)
@@ -1188,7 +1235,8 @@ class TestSignalCompleteness:
def test_the_fused_module_materializes_first_and_quants_last_across_chunks(self):
"""The FusedMoE shape: one module's copies span every owner-class chunk of
its group; materialize on its first, quant on its last."""
its group; materialize on its first, quant on its last.
"""
eng, names, group_lens = self._moe_planner(
worker_experts=[0, 2, 4, 6], n_layers=1
)
@@ -1246,7 +1294,8 @@ class TestLayerwiseGroups:
def test_flattening_a_partition_returns_the_input_order(self):
"""Un-indexed names group by POSITION, not by name class — so one that
looks like a pre-block name still lands after the layers it follows.
Group index therefore means the same thing on every rank."""
Group index therefore means the same thing on every rank.
"""
names = [
"embed.weight",
"model.layers.0.a",
@@ -1264,7 +1313,8 @@ class TestLayerwiseGroups:
def test_stacks_keep_prefix_appearance_order_and_sort_within(self):
"""Two stacks stay in the order they appear — the vision tower before the
text stack — while each sorts by index internally."""
text stack — while each sorts by index internally.
"""
names = [
"visual.blocks.1.w",
"visual.blocks.0.w",
@@ -1281,7 +1331,8 @@ class TestLayerwiseGroups:
def test_the_post_block_lands_last_however_early_it_arrives(self):
"""Megatron-Bridge streams the last pipeline stage's output block before
its layers. Sweeping un-indexed names after the first layer into a
trailing group is what lets the gather loop walk groups ascending."""
trailing group is what lets the gather loop walk groups ascending.
"""
names = ["model.layers.0.w", "model.norm.weight", "model.layers.1.w"]
assert layerwise_groups(names) == [
["model.layers.0.w"],
@@ -1291,7 +1342,8 @@ class TestLayerwiseGroups:
def test_names_sharing_a_layer_coalesce_when_the_source_interleaves(self):
"""A raw-checkpoint source may yield in shard-packing order. Coalescing on
the key keeps that from shattering one layer into many groups."""
the key keeps that from shattering one layer into many groups.
"""
names = [
"model.layers.0.a",
"model.layers.1.a",
@@ -1333,13 +1385,15 @@ class TestLayerwiseGroups:
def test_every_supported_naming_convention_partitions_per_layer(self, prefix):
"""One decoder layer per group under any prefix, with no per-architecture
table. A literal `model.layers.` match silently yields one whole-model
group for most of these."""
group for most of these.
"""
names = [f"{prefix}{i}.self_attn.q_proj.weight" for i in range(3)]
assert layerwise_groups(names) == [[n] for n in names]
def test_a_moe_layer_stays_whole(self):
"""The OUTERMOST index wins, so per-expert names group by layer. Splitting
on the expert would cut FusedMoE — one leaf module — across groups."""
on the expert would cut FusedMoE — one leaf module — across groups.
"""
names = [
"model.layers.0.mlp.experts.0.w1.weight",
"model.layers.0.mlp.experts.1.w1.weight",
@@ -1356,7 +1410,8 @@ class TestLayerwiseGroups:
def test_a_vlm_partitions_the_vision_tower_and_the_text_stack(self):
"""Qwen-VL layout. Today's literal `model.layers.` match puts every one of
these in ONE group — the whole model — because the text stack ships as
`model.language_model.layers.`."""
`model.language_model.layers.`.
"""
names = [
"visual.patch_embed.proj.weight",
"visual.blocks.0.attn.qkv.weight",
@@ -1375,7 +1430,8 @@ class TestLayerwiseGroups:
def test_two_stacks_sharing_an_index_do_not_merge(self):
"""The key carries the prefix, so vision block 0 and text layer 0 are
different groups even though both are index 0."""
different groups even though both are index 0.
"""
names = [
"visual.blocks.0.attn.qkv.weight",
"model.language_model.layers.0.self_attn.q_proj.weight",
@@ -1412,20 +1468,23 @@ class TestOpAllowlistAgreement:
def test_every_bake_recordable_op_is_replayable(self):
"""Anything the bake can record must be serveable, or the failure lands a
whole sync later than the mistake."""
whole sync later than the mistake.
"""
assert not set(SUPPORTED_OPS.values()) - set(ALLOWED_OPS)
def test_the_producer_allows_nothing_the_consumer_cannot_emit(self):
"""The allowlist is also a guard against a misbehaving or spoofed
consumer invoking arbitrary methods on trainer tensors, so it must not be
wider than the ops the bake can actually produce. ``to`` in particular
would let a replay change dtype or device."""
would let a replay change dtype or device.
"""
assert not set(ALLOWED_OPS) - set(SUPPORTED_OPS.values())
assert "to" not in ALLOWED_OPS
def test_transpose_via_t_is_serveable(self):
"""The specific regression: a chain recorded from ``.t()`` must pass the
producer's guard."""
producer's guard.
"""
fake = FakeRDTTensor(
name="w",
shape=torch.Size((4, 6)),
@@ -1470,7 +1529,8 @@ BAKE_MODELS = [
class TestRequiresTheRayExecutor:
"""The engine's data plane is Ray's, so a non-Ray executor cannot work. The
check is at construction because the alternative is an opaque failure during
the first handshake, long after the misconfiguration."""
the first handshake, long after the misconfiguration.
"""
def _construct(self, backend):
cfg = SimpleNamespace(
@@ -1497,7 +1557,8 @@ class TestRequiresTheRayExecutor:
def test_a_custom_executor_class_is_left_alone(self):
"""A `type[Executor]` override is deliberate and unjudgeable here, so it
must not be rejected for merely not being the string "ray"."""
must not be rejected for merely not being the string "ray".
"""
class _CustomExecutor:
pass
+79 -40
View File
@@ -236,7 +236,8 @@ class TestPublishAndRebuild:
def test_views_of_one_storage_do_not_overlap(self, server_factory):
"""One IPC export per storage, one as_strided view per name — the whole
point of the storage/view split. Overlapping views would serve the same
bytes for two different weights."""
bytes for two different weights.
"""
server = server_factory()
server.begin_sync(1)
_publish(server, GI_A, GROUP_A, nbytes=64)
@@ -259,7 +260,8 @@ class TestPublishAndRebuild:
def test_the_default_lookahead_is_one(self):
"""1 is the sweet spot: group N+1 is gathered AND pullable while N is
being pulled, with resident memory at its floor of 2 groups — the bound
the larger-model runs size against."""
the larger-model runs size against.
"""
assert DEFAULT_GATHER_LOOKAHEAD == 1
@@ -274,7 +276,8 @@ class TestFreeBarrier:
def test_group_is_held_until_the_last_live_consumer_signals(self, server_factory):
"""Every live consumer signals every owner; releasing on the first
signal would drop storage other consumers are still reading."""
signal would drop storage other consumers are still reading.
"""
server = server_factory()
server.begin_sync(3)
_publish(server, GI_A, GROUP_A)
@@ -301,7 +304,8 @@ class TestFreeBarrier:
):
"""A consumer with nothing to pull from a group signals it at sync
start, which can precede the publish. The publish must then release the
group rather than wait for a signal that will never come again."""
group rather than wait for a signal that will never come again.
"""
server = server_factory()
server.begin_sync(1)
server.free_group(GI_A)
@@ -324,7 +328,8 @@ class TestFreeBarrier:
the server says the group is gone — and it hears that ONLY through
``wait_freed`` / ``end_sync``, never a publish return (a freed notice
riding an unharvested async publish while the engine blocks in
``wait_freed`` would wedge the loop)."""
``wait_freed`` would wedge the loop).
"""
server = server_factory()
server.begin_sync(1)
_publish(server, GI_A, GROUP_A)
@@ -353,7 +358,8 @@ class TestFreeBarrier:
class _CountingSource:
"""`iter_groups` yielding one CPU-tensor group per call, built lazily so a
group's tensors exist only once the engine's credit gate lets it gather."""
group's tensors exist only once the engine's credit gate lets it gather.
"""
def __init__(self, groups):
self._groups = groups
@@ -366,7 +372,8 @@ class _CountingSource:
class _ResidencyDict(dict):
"""The engine's `_inflight`, instrumented: `max_len` is the high-water mark
of groups whose CUDA-IPC export refs were alive at once — the trainer's
resident-memory bound in groups."""
resident-memory bound in groups.
"""
def __init__(self):
super().__init__()
@@ -380,7 +387,8 @@ class _ResidencyDict(dict):
def _loop_engine(server, n_groups, *, lookahead):
"""A real engine wired straight to a real (non-Ray) server, with only the
state `_run_gather_loop` reads. The `_rpc` seam dispatches inline;
`_publish_async` sees no `.remote` and runs inline too."""
`_publish_async` sees no `.remote` and runs inline too.
"""
groups = [(f"model.layers.{gi}.w",) for gi in range(n_groups)]
e = ShardedRDTTrainerWeightTransferEngine.__new__(
ShardedRDTTrainerWeightTransferEngine
@@ -403,14 +411,16 @@ class TestGatherCredit:
serveable immediately, and the ENGINE's loop stops gathering while more than
`gather_lookahead` groups are unfreed. So at most `lookahead + 1` groups are
resident — two at the default — while group N+1 is pulled the instant N's
pulls finish."""
pulls finish.
"""
@pytest.fixture
def gather_engine(self, monkeypatch):
"""Neutralize the two CUDA touches in the export path so the REAL
gather loop runs on CPU: `.cuda()` becomes identity, and the storage
export ships the `(nbytes, ..., device, ...)` tuple the fixture's
stubbed `rebuild_cuda_tensor` expects."""
stubbed `rebuild_cuda_tensor` expects.
"""
monkeypatch.setattr(torch.Tensor, "cuda", lambda self: self)
monkeypatch.setattr(
trainer_mod,
@@ -422,7 +432,8 @@ class TestGatherCredit:
def test_publish_never_blocks(self, server_factory):
"""Publishing must not park on a credit: the whole point is that a
gathered group becomes serveable immediately. Three publishes with no
frees, inline on this thread — a block here hangs the test."""
frees, inline on this thread — a block here hangs the test.
"""
server = server_factory(gather_lookahead=1)
server.begin_sync(1)
for gi in range(3):
@@ -436,7 +447,8 @@ class TestGatherCredit:
than 2 groups of gathered weights, no matter how the consumer paces —
and the pipeline still overlaps, because the consumer here refuses to
free group N until N+1 is already published (pullable). A gate an
off-by-one too tight deadlocks this test; too loose fails max_len."""
off-by-one too tight deadlocks this test; too loose fails max_len.
"""
server = server_factory(gather_lookahead=1)
n_groups = 8
engine = gather_engine(server, n_groups, lookahead=1)
@@ -478,7 +490,8 @@ class TestGatherCredit:
):
"""0 is the serialized baseline: the loop waits for group N to be freed
before gathering N+1, so exactly ONE group is ever resident and nothing
overlaps. A floor of 1 on the bound would silently pipeline instead."""
overlaps. A floor of 1 on the bound would silently pipeline instead.
"""
server = server_factory(gather_lookahead=0)
engine = gather_engine(server, 4, lookahead=0)
@@ -503,8 +516,9 @@ class TestGatherCredit:
def test_the_residency_bound_scales_with_the_lookahead(
self, server_factory, gather_engine
):
"""lookahead + 1, not a hardcoded 2: at lookahead=2 a free-nothing
consumer sees exactly 3 groups gathered before the loop parks."""
"""Lookahead + 1, not a hardcoded 2: at lookahead=2 a free-nothing
consumer sees exactly 3 groups gathered before the loop parks.
"""
server = server_factory(gather_lookahead=2)
engine = gather_engine(server, 8, lookahead=2)
@@ -540,7 +554,8 @@ class TestGatherCredit:
def test_a_gather_error_releases_a_blocked_wait_freed(self, server_factory):
"""Otherwise a trainer-side failure on another rank deadlocks this one
inside its credit gate. It must RAISE, not return empty — an empty
return would spin the engine straight back into the wait."""
return would spin the engine straight back into the wait.
"""
server = server_factory()
server.begin_sync(1)
_publish(server, GI_A, GROUP_A)
@@ -621,7 +636,8 @@ class TestBeginSync:
def test_the_target_is_floored_at_one(self, server_factory):
"""A bare/zero call must not make every group free instantly (or divide
the barrier by zero)."""
the barrier by zero).
"""
server = server_factory()
server.begin_sync(0)
assert server._live_count == 1
@@ -629,7 +645,8 @@ class TestBeginSync:
def test_a_degraded_sync_lowers_the_target(self, server_factory):
"""FT degraded sync: the live count is the WHOLE degraded-sync
mechanism on this side — the group releases after the live consumers
alone."""
alone.
"""
server = server_factory()
server.begin_sync(2) # 4 provisioned, 2 alive
_publish(server, GI_A, GROUP_A)
@@ -653,7 +670,8 @@ class TestBeginSync:
def test_a_straggler_signal_cannot_credit_the_next_sync(self, server_factory):
"""begin_sync resets the counts, so a signal that leaked past the sync
boundary would over-credit — the reason the consumer drains its fired
signals before finishing. Pin the reset side of that contract."""
signals before finishing. Pin the reset side of that contract.
"""
server = server_factory()
server.begin_sync(2)
_publish(server, GI_A, GROUP_A)
@@ -675,7 +693,8 @@ class TestBeginSync:
self, server_factory
):
"""The packed layout repeats every sync — that is what makes caching the
destination views worth ~5ms per 384-spec group."""
destination views worth ~5ms per 384-spec group.
"""
server = server_factory()
server._pack_dsts[("sentinel",)] = (0, [])
server.begin_sync(1)
@@ -686,7 +705,8 @@ class TestServedNamesGuard:
"""A producer serves only the names it holds: its stage's groups, and
within them its own EP coordinate's experts plus the replicated names.
Without this guard a misrouted pull would block forever in the cache wait
for a name this rank never publishes."""
for a name this rank never publishes.
"""
def test_a_pull_for_an_unserved_name_fails_loudly(self, server_factory):
server = server_factory(served_names=list(GROUP_A))
@@ -702,7 +722,8 @@ class TestServedNamesGuard:
def test_served_names_none_accepts_anything(self, server_factory):
"""Gather-to-all: every producer holds every group, so there is nothing
to guard. The call proceeds to the cache wait (not exercised here)."""
to guard. The call proceeds to the cache wait (not exercised here).
"""
server = server_factory(served_names=None)
assert server._served_names is None
@@ -749,7 +770,8 @@ class TestStallWatchdog:
def test_end_sync_fails_when_a_consumer_never_signals(self, server_factory):
"""The exact mid-sync death signature: the group is published, one of
the two live consumers that owed a signal is gone."""
the two live consumers that owed a signal is gone.
"""
server = server_factory(stall_timeout_s=0.3)
server.begin_sync(2)
_publish(server, GI_A, GROUP_A)
@@ -763,7 +785,8 @@ class TestStallWatchdog:
"""The watchdog fires on the existing `set_gather_error` channel, so one
stall unwinds the whole rank through one path rather than each waiter
timing out separately — here both a consumer parked in the serve cache
wait and the engine parked in its credit gate."""
wait and the engine parked in its credit gate.
"""
server = server_factory(gather_lookahead=1, stall_timeout_s=0.3)
server.begin_sync(1)
_publish(server, GI_A, GROUP_A)
@@ -796,7 +819,8 @@ class TestStallWatchdog:
"""A consumer that is slow but signaling must never trip it: the stamp
is global to the producer, so a steady trickle of signals holds the
whole rank open. Nine sequential publishes with a 0.3s timeout each take
longer in total than the timeout, and none may fire."""
longer in total than the timeout, and none may fire.
"""
server = server_factory(gather_lookahead=1, stall_timeout_s=0.3)
server.begin_sync(1)
for gi in range(9):
@@ -809,7 +833,8 @@ class TestStallWatchdog:
def test_begin_sync_resets_the_progress_stamp(self, server_factory):
"""Syncs are minutes apart. Without the reset the first publish of sync N+1
would measure its stall from somewhere inside sync N and fire immediately."""
would measure its stall from somewhere inside sync N and fire immediately.
"""
server = server_factory(gather_lookahead=1, stall_timeout_s=0.3)
server.begin_sync(1)
_publish(server, GI_A, GROUP_A)
@@ -831,7 +856,8 @@ class TestConcurrentPublishAndFree:
self, server_factory
):
"""Every group published must end up freed and out of the cache, with
end_sync returning cleanly — the property whose violation is a hang."""
end_sync returning cleanly — the property whose violation is a hang.
"""
server = server_factory(gather_lookahead=2)
server.begin_sync(1)
groups = [(gi, (f"model.layers.{gi}.w",)) for gi in range(12)]
@@ -866,7 +892,8 @@ class TestConcurrentPublishAndFree:
):
"""All live consumers signaling at once: the release must happen on the
last signal and only once, or the engine drops IPC refs twice (or
never)."""
never).
"""
server = server_factory()
server.begin_sync(8)
_publish(server, GI_A, GROUP_A)
@@ -893,7 +920,8 @@ class TestFakeServerAgreesWithTheRealOne:
"""`test_sharded_rdt_trainer.py`'s `_FakeProducerServer` is a second,
independent model of this protocol that the engine-side tests assert on. Pin
the two against each other so the fake cannot silently drift from the
semantics the real server enforces."""
semantics the real server enforces.
"""
@staticmethod
def _fake():
@@ -924,7 +952,8 @@ class TestExportRing:
"""The packed export path. Its one hard invariant is that the trainer packs
a group exactly the way ``publish_group`` rebuilds it — a layout
disagreement would serve the wrong bytes with nothing downstream to catch
it."""
it.
"""
@staticmethod
def _engine():
@@ -1016,7 +1045,8 @@ class TestExportRingSlotSafety:
):
"""Free the later group of each pair first and hold the earlier one live
until the NEXT group is gathered -- the window a modular slot counter got
wrong (at lookahead=1 there are 2 slots, so group 2 took group 0's)."""
wrong (at lookahead=1 there are 2 slots, so group 2 took group 0's).
"""
server = server_factory(gather_lookahead=1)
n_groups = 8
engine = gather_engine(server, n_groups, lookahead=1)
@@ -1140,7 +1170,8 @@ class TestSharedServeSlots:
def test_one_deployment_never_shares(self, sharing_server):
"""With the width equal to the live count every group is a singleton, so
each call packs for itself out of its own ring."""
each call packs for itself out of its own ring.
"""
server = sharing_server(workers_per_replica=2)
server.begin_sync(2, [0, 1])
_publish(server, GI_A, GROUP_A)
@@ -1220,7 +1251,8 @@ class TestSharedServeSlots:
def test_a_seqless_call_is_refused(self, sharing_server):
"""The slot comes from ``seq``, so a caller that sends none cannot be
served safely."""
served safely.
"""
server = sharing_server(workers_per_replica=2)
server.begin_sync(1, [0])
_publish(server, GI_A, GROUP_A)
@@ -1230,7 +1262,8 @@ class TestSharedServeSlots:
def test_a_failed_pack_reaches_every_sharer(self, sharing_server):
"""Only the packer touches the buffer, so its failure has to be
republished: a waiter must raise rather than read a half-written slot."""
republished: a waiter must raise rather than read a half-written slot.
"""
server = sharing_server(workers_per_replica=2)
server.begin_sync(4, [0, 1, 2, 3])
_publish(server, GI_A, GROUP_A)
@@ -1254,7 +1287,8 @@ class TestSharedServeSlots:
def test_a_degraded_sync_narrows_the_rendezvous(self, sharing_server):
"""A dead deployment must not be waited for: a survivor left alone in
its group packs for itself instead of blocking on the corpse."""
its group packs for itself instead of blocking on the corpse.
"""
server = sharing_server(workers_per_replica=2, stall_timeout_s=30.0)
server.begin_sync(3, [0, 1, 3]) # consumer 2 died
_publish(server, GI_A, GROUP_A)
@@ -1268,7 +1302,8 @@ class TestSharedServeSlots:
def test_a_sharer_that_never_arrives_trips_the_watchdog(self, sharing_server):
"""A consumer that dies INSIDE the sync window has no detector, exactly
as for the free barrier; the rendezvous waits on the same channel."""
as for the free barrier; the rendezvous waits on the same channel.
"""
server = sharing_server(workers_per_replica=2, stall_timeout_s=0.3)
server.begin_sync(4, [0, 1, 2, 3])
_publish(server, GI_A, GROUP_A)
@@ -1280,7 +1315,8 @@ class TestSharedServeSlots:
class TestSharedReservation:
"""``reserve_serve_buffer`` under sharing: one ring per GROUP, and the
init-time check that the group's members pull the same chunks."""
init-time check that the group's members pull the same chunks.
"""
def test_matching_plans_reserve_one_ring_between_them(self, sharing_server):
server = sharing_server(workers_per_replica=2)
@@ -1302,7 +1338,8 @@ class TestSharedReservation:
def test_mismatched_plans_fail_at_reservation(self, sharing_server):
"""Sharing rests on the sharers pulling the same chunks. Checked here,
where it names both consumers, rather than at serve time, where it is a
rendezvous nobody completes."""
rendezvous nobody completes.
"""
server = sharing_server(workers_per_replica=2)
server.reserve_serve_buffer(0, 1024, "plan-a")
@@ -1311,7 +1348,8 @@ class TestSharedReservation:
def test_different_groups_may_differ(self, sharing_server):
"""Only the members of ONE group have to agree; worker 0 and worker 1 of
the same deployment hold different slices and never share a slot."""
the same deployment hold different slices and never share a slot.
"""
server = sharing_server(workers_per_replica=2)
server.reserve_serve_buffer(0, 1024, "plan-a")
server.reserve_serve_buffer(1, 1024, "plan-b")
@@ -1320,7 +1358,8 @@ class TestSharedReservation:
def test_a_missing_digest_skips_the_check(self, sharing_server):
"""The digest is optional on the wire, so an engine that does not send
one still reserves."""
one still reserves.
"""
server = sharing_server(workers_per_replica=2)
server.reserve_serve_buffer(0, 1024, None)
server.reserve_serve_buffer(2, 1024, None)
+24 -12
View File
@@ -34,7 +34,8 @@ from vllm.distributed.weight_transfer.sharded_rdt_trainer import (
class _ListSource(WeightSource):
"""A WeightSource over an explicit ordered (name, cpu-tensor) list, so the
sharded-RDT group/order logic can be tested without a real model."""
sharded-RDT group/order logic can be tested without a real model.
"""
def __init__(self, pairs):
self._pairs = list(pairs)
@@ -54,7 +55,8 @@ class _FakeProducerServer:
per-group barrier: publish/free are keyed by GROUP INDEX, signals count
to the ``begin_sync`` live total, publish_group returns nothing, and freed
groups flow back only through wait_freed / end_sync (see
test_sharded_rdt_producer.TestFakeServerAgreesWithTheRealOne)."""
test_sharded_rdt_producer.TestFakeServerAgreesWithTheRealOne).
"""
def __init__(self, auto_free=True):
self.order: list[str] = []
@@ -98,7 +100,8 @@ class _FakeProducerServer:
must already have a banked credit when the gate asks (auto_free, or a
test's own free_group) — anything else is the deadlock the real
watchdog would kill, so fail loudly. Not appended to ``order``: it is
pacing, not a lifecycle milestone."""
pacing, not a lifecycle milestone.
"""
assert self._pending_freed, (
"wait_freed with nothing freed: the gather loop would deadlock"
)
@@ -121,7 +124,8 @@ def _rdt_engine_with_fake_server(
):
"""Build a ShardedRDTTrainerWeightTransferEngine wired to an in-process fake
server (no Ray, no CUDA IPC): bypass trainer_init's spawn, set the
group-major metadata, and route _rpc to the fake."""
group-major metadata, and route _rpc to the fake.
"""
import vllm.distributed.weight_transfer.sharded_rdt_trainer as mod
# reduce_tensor needs CUDA; the fake server never rebuilds, so stub it.
@@ -321,7 +325,8 @@ def test_sharded_rdt_send_weights_surfaces_update_error(monkeypatch):
class _OwnedSource(_ListSource):
"""A source holding only some groups' names, like a pipeline-parallel rank.
Takes group indices for convenience and declares the names inside them."""
Takes group indices for convenience and declares the names inside them.
"""
def __init__(self, pairs, owned_group_idx):
super().__init__(pairs)
@@ -387,7 +392,8 @@ def test_sharded_rdt_owned_group_order_mismatch_raises(monkeypatch):
)
def test_sharded_rdt_begin_sync_carries_the_live_count(monkeypatch):
"""The free barrier's target is one integer per sync: the live consumer
count, defaulting to the whole provisioned fleet."""
count, defaulting to the whole provisioned fleet.
"""
server = _FakeProducerServer(auto_free=True)
engine = _rdt_engine_with_fake_server(
_rdt_source_two_layers(),
@@ -405,7 +411,8 @@ def test_sharded_rdt_begin_sync_carries_the_live_count(monkeypatch):
class TestLiveCountPlumbing:
"""``send_weights(live_consumer_ids)`` -> the barrier target. The
provisioned geometry is frozen; a degraded sync only lowers the target and
narrows each slot-sharing group's rendezvous."""
narrows each slot-sharing group's rendezvous.
"""
@staticmethod
def _engine(num_consumers, world=2, rank=0):
@@ -438,7 +445,8 @@ class TestLiveCountPlumbing:
def test_which_consumers_died_does_not_matter_only_how_many(self):
"""The whole point of the barrier: no routed per-producer targets, so
the identity of the dead consumer is irrelevant to the producers."""
the identity of the dead consumer is irrelevant to the producers.
"""
counts = []
for live in ([0, 1, 2, 3], [4, 5, 6, 7], [0, 2, 4, 6]):
engine, got = self._engine(num_consumers=8)
@@ -449,7 +457,8 @@ class TestLiveCountPlumbing:
def test_the_live_ids_travel_with_the_count(self):
"""The count sizes the free barrier, the ids size the slot-sharing
rendezvous, so a producer that shares slots can tell WHICH consumers it
is still waiting for. They must describe the same set."""
is still waiting for. They must describe the same set.
"""
engine, got = self._engine(num_consumers=8)
engine.send_weights([6, 0, 2, 2])
((count, ids),) = got
@@ -517,7 +526,8 @@ def test_sharded_rdt_rejects_a_rank_holding_nothing(monkeypatch):
def test_sharded_rdt_rejects_a_name_no_rank_holds(monkeypatch):
"""Every name must be held somewhere or it can never be served — caught
when the holdings are transposed, naming the orphan."""
when the holdings are transposed, naming the orphan.
"""
with pytest.raises(ValueError, match="no trainer rank holds"):
_rdt_engine_with_fake_server(
_OwnedSource(_rdt_source_two_layers()._pairs, [0, 1]),
@@ -531,7 +541,8 @@ def test_sharded_rdt_rejects_a_name_no_rank_holds(monkeypatch):
def test_sharded_rdt_rejects_metadata_disagreement_across_ranks(monkeypatch):
"""Only the sender's metadata reaches the consumers, so a rank describing
just its own share must fail loudly rather than silently drop the rest."""
just its own share must fail loudly rather than silently drop the rest.
"""
engine = _rdt_engine_with_fake_server(
_OwnedSource(_rdt_source_two_layers()._pairs, [1, 2]),
is_sender=False,
@@ -554,7 +565,8 @@ def _serve_ring_server(src_name, src):
"""A producer server with one cached tensor and a pre-seeded serve ring, so a
pull needs no Ray and no NIXL registration. Returns (server, serve) where
``serve(chain)`` packs one spec into the SAME ring slot every time — which is
what puts the destination-view cache, and only it, under test."""
what puts the destination-view cache, and only it, under test.
"""
from vllm.distributed.weight_transfer.sharded_rdt_trainer import (
_RDTProducerServer,
)
+8 -6
View File
@@ -43,6 +43,7 @@ def distributed_run(fn, world_size, timeout=60):
fn: Function to run in each process
world_size: Number of processes to spawn
timeout: Maximum time in seconds to wait for processes (default: 60)
"""
number_of_processes = world_size
processes = []
@@ -418,7 +419,8 @@ def test_tensor_broadcast():
def _dumps_oob(obj) -> tuple[bytes, list]:
"""Pickle `obj` the same way `MessageQueue.enqueue` does: tensor
dispatch table + out-of-band buffers >= 1MiB."""
dispatch table + out-of-band buffers >= 1MiB.
"""
buffers = []
def callback(buf: pickle.PickleBuffer) -> bool:
@@ -489,7 +491,8 @@ def test_tensor_pickle_roundtrip(case: str):
@pytest.mark.parametrize("case", ["cuda", "requires_grad", "conj"])
def test_reduce_tensor_fallback(case: str):
"""Tensors the zero-copy reducer can't safely alias must fall back to
torch's default reduction."""
torch's default reduction.
"""
if case == "cuda":
if not torch.cuda.is_available():
pytest.skip("requires CUDA")
@@ -634,11 +637,9 @@ def test_acquire_read_releases_slot_when_reader_raises():
def test_warning_logs(caplog_vllm):
"""
Test that warning logs are emitted at VLLM_RINGBUFFER_WARNING_INTERVAL intervals
"""Test that warning logs are emitted at VLLM_RINGBUFFER_WARNING_INTERVAL intervals
when indefinite=False, and are not emitted when indefinite=True.
"""
# Patch the warning log interval to every 1 ms during reads
with mock.patch(
"vllm.distributed.device_communicators.shm_broadcast.VLLM_RINGBUFFER_WARNING_INTERVAL",
@@ -754,7 +755,8 @@ def test_remote_subscribe_addr_unique_concurrent_writers(
Pre-fix, the writer probed a port with get_open_port() and bound it
afterwards; pinning the probe to one free port makes every writer
bind the same port and fail deterministically on that code path,
while the late-binding implementation never consults the probe."""
while the late-binding implementation never consults the probe.
"""
from vllm.distributed.device_communicators import shm_broadcast
colliding_port = get_open_port()
+9 -9
View File
@@ -12,20 +12,20 @@ from vllm.distributed.device_communicators.shm_object_storage import (
class TestSingleWriterShmRingBuffer(unittest.TestCase):
"""Test suite for the ring buffer implementation"""
"""Test suite for the ring buffer implementation."""
def setUp(self):
"""Set up test fixtures"""
"""Set up test fixtures."""
self.buffer_size = 4096
self.ring_buffer = None
def tearDown(self):
"""Clean up after tests"""
"""Clean up after tests."""
if self.ring_buffer:
self.ring_buffer.close()
def test_buffer_opening(self):
"""Test opening an existing buffer"""
"""Test opening an existing buffer."""
# First create a buffer
self.ring_buffer = SingleWriterShmRingBuffer(
data_buffer_size=self.buffer_size, create=True
@@ -39,7 +39,7 @@ class TestSingleWriterShmRingBuffer(unittest.TestCase):
)
def test_buffer_access(self):
"""Test accessing allocated buffers"""
"""Test accessing allocated buffers."""
self.ring_buffer = SingleWriterShmRingBuffer(
data_buffer_size=self.buffer_size, create=True
)
@@ -61,7 +61,7 @@ class TestSingleWriterShmRingBuffer(unittest.TestCase):
self.assertEqual(read_id, monotonic_id)
def test_memory_error_on_full_buffer(self):
"""Test that MemoryError is raised when buffer is full"""
"""Test that MemoryError is raised when buffer is full."""
small_buffer_size = 200
self.ring_buffer = SingleWriterShmRingBuffer(
data_buffer_size=small_buffer_size, create=True
@@ -76,7 +76,7 @@ class TestSingleWriterShmRingBuffer(unittest.TestCase):
self.ring_buffer.allocate_buf(1) # Would exceed buffer capacity
def test_allocation_and_free(self):
"""Test allocation and freeing of buffers"""
"""Test allocation and freeing of buffers."""
small_buffer_size = 200
self.ring_buffer = SingleWriterShmRingBuffer(
data_buffer_size=small_buffer_size, create=True
@@ -96,7 +96,7 @@ class TestSingleWriterShmRingBuffer(unittest.TestCase):
self.assertEqual(freed_ids[0], i)
def test_clear_buffer(self):
"""Test clearing the buffer"""
"""Test clearing the buffer."""
self.ring_buffer = SingleWriterShmRingBuffer(
data_buffer_size=self.buffer_size, create=True
)
@@ -180,7 +180,7 @@ class TestSingleWriterShmRingBuffer(unittest.TestCase):
def main():
"""Main function demonstrating usage and running tests"""
"""Main function demonstrating usage and running tests."""
print("=== SingleWriterShmRingBuffer Test Suite ===\n")
# Run unit tests
+2 -1
View File
@@ -31,7 +31,8 @@ class _CUDADeviceCountStatelessTestActor:
def test_cuda_device_count_stateless():
"""Test that cuda_device_count_stateless changes return value if
CUDA_VISIBLE_DEVICES is changed."""
CUDA_VISIBLE_DEVICES is changed.
"""
if current_platform.is_rocm():
pytest.skip("Skip for ROCm because Ray uses HIP_VISIBLE_DEVICES.")
actor = _CUDADeviceCountStatelessTestActor.options( # type: ignore
+44 -22
View File
@@ -1166,7 +1166,8 @@ class RecordingClient:
def _module_with(*pairs):
"""A tiny nn.Module exposing the given (name, tensor) pairs as parameters,
so trainer tests can build a ModuleSource without a real model."""
so trainer tests can build a ModuleSource without a real model.
"""
module = torch.nn.Module()
for name, tensor in pairs:
module.register_parameter(name, torch.nn.Parameter(tensor, requires_grad=False))
@@ -1287,7 +1288,8 @@ class TestModuleSource:
def test_metadata_agrees_with_iteration(self):
"""The two channels must line up element-for-element: engines declare
the round from `metadata()` and then send what iteration yields."""
the round from `metadata()` and then send what iteration yields.
"""
source = ModuleSource(
_module_with(("w", torch.zeros(2, 3)), ("b", torch.zeros(3)))
)
@@ -1301,11 +1303,13 @@ class TestModuleSource:
class TestWeightSourceGroupContract:
"""`groups()` / `iter_groups()` on the WeightSource ABC. Groups are what
backends gather and free by, so the default must agree with
`layerwise_groups` over `metadata()`, restricted to what this rank holds."""
`layerwise_groups` over `metadata()`, restricted to what this rank holds.
"""
class _Source(WeightSource):
"""Minimal source over an ordered (name, tensor) list, optionally holding
only some names (in which case it iterates only their groups)."""
only some names (in which case it iterates only their groups).
"""
def __init__(self, names, held=None, reverse=False):
self._pairs = [(n, torch.full((2,), float(i))) for i, n in enumerate(names)]
@@ -1360,7 +1364,8 @@ class TestWeightSourceGroupContract:
def test_groups_order_follows_metadata_not_the_declaration(self):
"""The declaration is a SET of names; the groups it selects still come
out in metadata order, so each pairs with the right ``iter_groups()``
batch however the source listed them."""
batch however the source listed them.
"""
names = ["embed.w", "model.layers.0.a", "model.layers.1.a", "norm.w"]
source = self._source(
names, held=["model.layers.1.a", "model.layers.0.a", "model.layers.1.a"]
@@ -1394,14 +1399,16 @@ class TestWeightSourceGroupContract:
def test_out_of_order_iteration_raises(self):
"""Materializing is usually a collective, so a rank that iterates out of
order deadlocks its peers -- fail loudly instead."""
order deadlocks its peers -- fail loudly instead.
"""
source = self._source(["model.layers.0.a", "model.layers.0.b"], reverse=True)
with pytest.raises(RuntimeError, match="iteration order must match"):
list(source.iter_groups())
def test_a_source_may_override_iter_groups(self):
"""The extension point: materialize a whole group in one step instead of
one generator resume per tensor."""
one generator resume per tensor.
"""
calls = []
class _Batched(TestWeightSourceGroupContract._Source):
@@ -1422,7 +1429,8 @@ class TestDeferredProcessingContract:
caller that takes over the update tail (running its own
`finalize_layerwise_reload` instead of going through `finish_weight_update`)
reads the flag and calls the method. Both must be answerable on any engine, or
that caller ends up reaching through a getattr."""
that caller ends up reaching through a getattr.
"""
def _engines(self):
return {
@@ -1436,7 +1444,8 @@ class TestDeferredProcessingContract:
def test_every_engine_can_be_drained(self):
"""The default is a no-op, so a caller never has to check whether the
method exists before calling it."""
method exists before calling it.
"""
for name, cls in self._engines().items():
assert callable(cls.drain_pending), name
@@ -1530,7 +1539,8 @@ class TestTrainerEngineBase:
)
def test_ipc_trainer_send_weights_drives_client_in_order():
"""send_weights issues start -> update -> finish and ships per-round metadata;
the packed wire param rides the init info, not the per-round update_info."""
the packed wire param rides the init info, not the per-round update_info.
"""
client = RecordingClient()
engine = IPCTrainerWeightTransferEngine(
client=client,
@@ -1549,7 +1559,8 @@ def test_ipc_trainer_send_weights_drives_client_in_order():
def test_ipc_trainer_init_ships_packed_to_worker():
"""trainer_init drives the inference-side init handshake and propagates the
must-agree `packed` flag to the worker."""
must-agree `packed` flag to the worker.
"""
if torch.accelerator.device_count() < 1:
pytest.skip("Need at least 1 GPU (CUDA IPC handles).")
@@ -1570,7 +1581,8 @@ def test_ipc_trainer_init_ships_packed_to_worker():
def test_nccl_trainer_init_ships_worker_init_info(monkeypatch):
"""The sender's trainer_init drives the inference-side init handshake with
the worker-shaped init info (rank_offset=1) while opening its own endpoint,
and propagates the must-agree wire params to the worker."""
and propagates the must-agree wire params to the worker.
"""
import vllm.distributed.weight_transfer.nccl_engine as nccl_engine_mod
# Bypass the real NCCL rendezvous.
@@ -1611,7 +1623,8 @@ def test_nccl_trainer_init_ships_worker_init_info(monkeypatch):
def test_nccl_worker_learns_wire_params_from_init_handshake(monkeypatch):
"""The worker engine reads packed + buffer geometry from the
trainer-supplied init info at the handshake, not from the config or the
per-round update info."""
per-round update info.
"""
import vllm.distributed.weight_transfer.nccl_engine as nccl_engine_mod
monkeypatch.setattr(
@@ -1644,7 +1657,8 @@ def test_nccl_worker_learns_wire_params_from_init_handshake(monkeypatch):
def test_nccl_trainer_init_non_sender_skips_rendezvous_and_client():
"""Non-sender trainer ranks build an engine without opening an endpoint or
touching the client; they only join the collectives in send_weights."""
touching the client; they only join the collectives in send_weights.
"""
client = RecordingClient()
engine = WeightTransferTrainerFactory.trainer_init(
init_info=NCCLTrainerInitInfo(
@@ -1674,7 +1688,8 @@ def test_nccl_trainer_init_non_sender_skips_rendezvous_and_client():
def test_nccl_trainer_send_weights_drives_client_in_order():
"""send_weights issues start -> update -> finish and ships per-round
metadata; the packed wire params ride the init handshake, not the
per-round update_info."""
per-round update_info.
"""
client = RecordingClient()
engine = NCCLTrainerWeightTransferEngine(
client=client,
@@ -1695,7 +1710,8 @@ def test_nccl_trainer_send_weights_drives_client_in_order():
class _ScriptedSource(WeightSource):
"""Declares `meta` but yields whatever `pairs` says — used to drive the
metadata/iteration agreement checks."""
metadata/iteration agreement checks.
"""
def __init__(self, meta, pairs):
self._meta = meta
@@ -1731,7 +1747,8 @@ def test_nccl_trainer_init_requires_source():
def test_nccl_trainer_send_weights_rejects_reordered_source(monkeypatch):
"""The worker sizes its buffers (and cuts packed chunks) from metadata(), so
iteration disagreeing with it must raise rather than corrupt the stream."""
iteration disagreeing with it must raise rather than corrupt the stream.
"""
meta = [
ParamMeta("w", torch.float32, (4,)),
ParamMeta("b", torch.float32, (2,)),
@@ -1745,7 +1762,8 @@ def test_nccl_trainer_send_weights_rejects_reordered_source(monkeypatch):
def test_nccl_trainer_send_weights_rejects_dtype_disagreement(monkeypatch):
"""A source that declares one wire dtype and materializes another would make
the two sides disagree on every byte offset."""
the two sides disagree on every byte offset.
"""
meta = [ParamMeta("w", torch.float32, (4,))]
engine = _mock_group_engine(
_ScriptedSource(meta, [("w", torch.zeros(4, dtype=torch.bfloat16))]),
@@ -1772,7 +1790,8 @@ def test_nccl_trainer_send_weights_rejects_truncated_source(monkeypatch):
def test_nccl_trainer_send_weights_broadcasts_contiguous(monkeypatch):
"""NCCL sends numel elements from data_ptr(), so a non-contiguous view must
be linearized first or the worker receives unrelated memory."""
be linearized first or the worker receives unrelated memory.
"""
base = torch.arange(6, dtype=torch.float32).reshape(2, 3)
view = base.t() # non-contiguous
meta = [ParamMeta("w", torch.float32, tuple(view.shape))]
@@ -1842,7 +1861,8 @@ def _sparse_patch(device: str = "cpu") -> SparseWeightPatch:
def test_sparse_nccl_trainer_init_ships_worker_init_info(monkeypatch):
"""The sender's trainer_init drives the init handshake with the
worker-shaped init info; sparse ships no packed wire params, so the worker
keeps its unpacked defaults. Sparse takes no `source`."""
keeps its unpacked defaults. Sparse takes no `source`.
"""
import vllm.distributed.weight_transfer.sparse_nccl_engine as sparse_mod
monkeypatch.setattr(sparse_mod, "open_trainer_endpoint", lambda info: MagicMock())
@@ -1895,7 +1915,8 @@ def test_sparse_nccl_trainer_send_weights_drives_client_in_order(monkeypatch):
def test_sparse_nccl_trainer_send_weights_empty_round_is_noop():
"""A round with no patches must not touch the client (an empty sparse
update info is invalid by construction)."""
update info is invalid by construction).
"""
client = RecordingClient()
engine = SparseNCCLTrainerWeightTransferEngine(client=client)
engine.model_update_group = MagicMock()
@@ -1931,7 +1952,8 @@ def test_sparse_nccl_trainer_rejects_source():
def test_sparse_nccl_trainer_validates_patch_before_any_rpc():
"""Malformed patches must fail on the trainer, before start_weight_update:
the worker's own checks only run once the broadcasts are already in flight,
where a size mismatch wedges both sides instead of raising."""
where a size mismatch wedges both sides instead of raising.
"""
client = RecordingClient()
engine = SparseNCCLTrainerWeightTransferEngine(client=client)
engine.model_update_group = MagicMock()
@@ -403,7 +403,8 @@ def trainer_broadcast_tensor_torch_free(
def _run_uid_transfer(trainer_task) -> None:
"""Mint a unique id, run the worker and the given trainer peer concurrently,
and assert the tensor round-trips. Both ranks must enter init together --
there is no store barrier on the UID path."""
there is no store barrier on the UID path.
"""
from vllm.distributed.device_communicators.pynccl_wrapper import NCCLLibrary
_init_ray_for_weight_transfer()
@@ -450,7 +451,8 @@ def _run_uid_transfer(trainer_task) -> None:
)
def test_nccl_weight_transfer_between_processes_uid():
"""Weight transfer over a pre-shared ncclUniqueId (no TCPStore), with both
ranks on vLLM's ``PyNcclCommunicator``."""
ranks on vLLM's ``PyNcclCommunicator``.
"""
_run_uid_transfer(trainer_broadcast_tensor_uid)
@@ -460,6 +462,7 @@ def test_nccl_weight_transfer_between_processes_uid():
)
def test_nccl_weight_transfer_torch_free_trainer():
"""UID weight transfer against a foreign peer that uses no torch and no vLLM
communicator, only a cupy buffer and raw ``ncclBroadcast``."""
communicator, only a cupy buffer and raw ``ncclBroadcast``.
"""
pytest.importorskip("cupy")
_run_uid_transfer(trainer_broadcast_tensor_torch_free)
+12 -10
View File
@@ -394,8 +394,7 @@ def test_media_io_kwargs_parser(arg, expected):
],
)
def test_optimization_level(args, expected):
"""
Test space-separated optimization levels (-O 1, -O 2, -O 3) map to
"""Test space-separated optimization levels (-O 1, -O 2, -O 3) map to
optimization_level.
"""
parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
@@ -414,9 +413,7 @@ def test_optimization_level(args, expected):
],
)
def test_mode_parser(args, expected):
"""
Test compilation config modes (-cc.mode=int) map to compilation_config.
"""
"""Test compilation config modes (-cc.mode=int) map to compilation_config."""
parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
parsed_args = parser.parse_args(args)
assert parsed_args.compilation_config.mode == expected
@@ -573,7 +570,8 @@ def test_attention_config():
def test_multi_node_world_size_includes_pcp(monkeypatch):
"""PCP expands the process world size, so the --nnodes divisibility check
must include it. Without this, TP=1/PCP=2 over 2 nodes computes a world
size of 1 and the launch is rejected before the engine starts."""
size of 1 and the launch is rejected before the engine starts.
"""
import vllm.config.vllm
# PCP requires the V2 model runner, which is gated on Triton.
@@ -799,7 +797,8 @@ def test_expand_json_human_readable_numbers(input_json, expected_json):
)
def test_cloud_storage_uri_skips_get_model_path(uri, monkeypatch):
"""Cloud storage URIs should not be passed to get_model_path()
when HF_HUB_OFFLINE=1, as they are not valid HF repo IDs."""
when HF_HUB_OFFLINE=1, as they are not valid HF repo IDs.
"""
import huggingface_hub
monkeypatch.setattr(huggingface_hub.constants, "HF_HUB_OFFLINE", True)
@@ -811,7 +810,8 @@ def test_cloud_storage_uri_skips_get_model_path(uri, monkeypatch):
def test_cloud_storage_tokenizer_skips_get_model_path(monkeypatch):
"""Cloud storage tokenizer URI should not be passed to
get_model_path() when HF_HUB_OFFLINE=1."""
get_model_path() when HF_HUB_OFFLINE=1.
"""
import huggingface_hub
monkeypatch.setattr(huggingface_hub.constants, "HF_HUB_OFFLINE", True)
@@ -973,7 +973,8 @@ class TestDpDeviceIdSharding:
def test_dp_supervisor_device_ids_stay_env_relative(self):
"""Regression test: the DP supervisor must pass env-relative indices,
not physical IDs, because each child re-resolves --device-ids
against its inherited device-control env var."""
against its inherited device-control env var.
"""
import argparse
from vllm.entrypoints.launchers.dp_supervisor import _build_device_ids
@@ -1000,7 +1001,8 @@ class TestDpDeviceIdSharding:
def test_dp_rank_shards_user_assigned_gpu_ids(self):
"""get_physical_gpu_ids_for_local_dp_rank slices the user-provided
--device-ids list instead of recomputing from the env var."""
--device-ids list instead of recomputing from the env var.
"""
from vllm.platforms import current_platform
from vllm.v1.engine.utils import get_physical_gpu_ids_for_local_dp_rank
@@ -241,7 +241,8 @@ class TestToolResultContent:
self, tool_result_content
) -> AnthropicMessagesRequest:
"""Build a request with assistant tool_use followed by user
tool_result."""
tool_result.
"""
return _make_request(
[
{
@@ -326,7 +327,8 @@ class TestToolResultContent:
def test_tool_result_with_text_and_image(self):
"""Mixed text+image tool_result: text in tool msg, image in user
msg."""
msg.
"""
request = self._make_tool_result_request(
[
{"type": "text", "text": "Here is the screenshot"},
@@ -423,7 +425,8 @@ class TestToolResultContent:
class TestAttributionHeaderStripping:
def test_billing_header_stripped_from_system(self):
"""Claude Code's x-anthropic-billing-header block should be
stripped to preserve prefix caching."""
stripped to preserve prefix caching.
"""
request = _make_request(
[{"role": "user", "content": "Hello"}],
system=[
@@ -480,7 +483,7 @@ class TestThinkingBlockConversion:
"""
def test_thinking_plus_text_in_assistant_message(self):
"""thinking + text → reasoning field + plain-string content."""
"""Thinking + text → reasoning field + plain-string content."""
request = _make_request(
[
{"role": "user", "content": "Write me some code."},
@@ -542,7 +545,7 @@ class TestThinkingBlockConversion:
assert asst.get("content") is None
def test_thinking_plus_tool_use_in_assistant_message(self):
"""thinking + tool_use: reasoning field set, tool_calls populated."""
"""Thinking + tool_use: reasoning field set, tool_calls populated."""
request = _make_request(
[
{"role": "user", "content": "What is 2+2?"},
@@ -1147,7 +1150,8 @@ class TestMessageStreamConverterToolUseContentBuffering:
async def test_buffered_content_flushed_on_done_without_usage_chunk(self):
"""Content buffered during tool_use must be emitted even if the
stream jumps straight from finish_reason to [DONE], skipping the
empty-choices usage chunk."""
empty-choices usage chunk.
"""
async def sse_input():
yield _make_stream_chunk(
@@ -1251,7 +1255,8 @@ class TestStreamingCacheUsageSemantics:
async def test_streaming_cache_fields_absent_then_populated(self):
"""First chunk lacks prompt_tokens_details (vLLM contract);
message_start omits cache fields. The final chunk carries
prompt_tokens_details, so message_delta carries resolved values."""
prompt_tokens_details, so message_delta carries resolved values.
"""
async def sse_input():
yield _make_stream_chunk(
@@ -1296,7 +1301,8 @@ class TestStreamingCacheUsageSemantics:
@pytest.mark.asyncio
async def test_streaming_no_cache_hit(self):
"""When the final chunk reports cached_tokens=0, message_delta carries
cache fields = 0 (cache miss); message_start still omits them."""
cache fields = 0 (cache miss); message_start still omits them.
"""
async def sse_input():
yield _make_stream_chunk(
@@ -1337,7 +1343,8 @@ class TestStreamingCacheUsageSemantics:
@pytest.mark.asyncio
async def test_streaming_no_prompt_tokens_details_at_all(self):
"""If --enable-prompt-tokens-details is off, no chunk carries cache
info; both message_start and message_delta omit cache fields."""
info; both message_start and message_delta omit cache fields.
"""
async def sse_input():
yield _make_stream_chunk(
@@ -1458,7 +1465,8 @@ class TestMessagesFullConverter:
class TestCacheSalt:
def test_cache_salt_passed_through(self):
"""cache_salt on the Anthropic request reaches the converted
ChatCompletionRequest so prefix-cache isolation works via /v1/messages."""
ChatCompletionRequest so prefix-cache isolation works via /v1/messages.
"""
request = _make_request(
[{"role": "user", "content": "Hello"}],
cache_salt="tenant-abc-secret-salt",
@@ -1663,7 +1671,8 @@ class TestClientErrorResponses:
@staticmethod
def _conversion_error() -> ValidationError:
"""A real pydantic ValidationError like the one ChatCompletionRequest
construction raises when Anthropic input violates the OpenAI schema."""
construction raises when Anthropic input violates the OpenAI schema.
"""
class _StubRequest(BaseModel):
stop: Annotated[list[str], Field(max_length=4)] | None = None
@@ -1674,7 +1683,8 @@ class TestClientErrorResponses:
def test_validation_error_returns_bad_request(self):
"""A pydantic ValidationError during Anthropic->OpenAI conversion is
surfaced as a 400 BadRequestError, not a 500."""
surfaced as a 400 BadRequestError, not a 500.
"""
handler = MagicMock(spec=AnthropicServingMessages)
handler.create_messages.side_effect = self._conversion_error()
@@ -87,7 +87,8 @@ def _make_chunk(
citations: list[Any] | None = None,
) -> str:
"""Build the ``data: {...}\\n\\n`` SSE frame the production code
consumes."""
consumes.
"""
delta: dict[str, Any] = {}
if role is not None:
delta["role"] = role
@@ -135,7 +136,8 @@ async def _stream_from(items: list[str]) -> AsyncGenerator[str, None]:
async def _drain(serving: CohereServingChatV2, items: list[str]) -> list[str]:
"""Drive ``_chat_completion_stream_to_v2`` over ``items`` and collect
the emitted SSE frames."""
the emitted SSE frames.
"""
request = CohereChatV2Request(
model="m", messages=[{"role": "user", "content": "hi"}], stream=True
)
@@ -50,7 +50,8 @@ def update_admission_stats_worker(listen_address, sock, args, client_config):
# importable by `multiprocessing.spawn` (no closures, no nesting).
def defer_addresses_stub_worker(listen_address, sock, args, client_config):
"""Bind ROUTER/PULL with a kernel-assigned port, report the actual
endpoints back via the pipe, then exit."""
endpoints back via the pipe, then exit.
"""
ctx = zmq.Context()
try:
in_sock = make_zmq_socket(
@@ -357,7 +358,8 @@ def test_external_process_monitoring(api_server_args):
def test_gather_actual_addresses_end_to_end():
"""Each child binds ROUTER/PULL with a kernel-picked port and reports
the bound endpoints back via its per-child pipe; the manager surfaces
them via :py:meth:`gather_actual_addresses`."""
them via :py:meth:`gather_actual_addresses`.
"""
host = "127.0.0.1"
num_servers = 4
@@ -407,7 +409,8 @@ def test_gather_actual_addresses_end_to_end():
@pytest.mark.timeout(30)
def test_gather_actual_addresses_child_crash_before_report():
"""A child that exits before sending its endpoints must surface a
clear ``RuntimeError`` rather than hang or return ``None`` slots."""
clear ``RuntimeError`` rather than hang or return ``None`` slots.
"""
host = "127.0.0.1"
num_servers = 2
placeholder_inputs = [
@@ -444,7 +447,8 @@ def test_gather_actual_addresses_child_crash_before_report():
def test_rust_frontend_launch_log_redacts_credentials(monkeypatch, caplog):
"""The Rust frontend command carries every non-default arg as JSON.
Credentials must not reach the log line."""
Credentials must not reach the log line.
"""
import subprocess as subprocess_mod
from vllm.entrypoints.launchers.cli_args import make_arg_parser
+17 -15
View File
@@ -152,7 +152,7 @@ def test_multiple_valid_inputs(serve_parser):
### Tests for serve argument validation that run prior to loading
def test_enable_auto_choice_passes_without_tool_call_parser(serve_parser):
"""Ensure validation fails if tool choice is enabled with no call parser"""
"""Ensure validation fails if tool choice is enabled with no call parser."""
# If we enable-auto-tool-choice, explode with no tool-call-parser
args = serve_parser.parse_args(args=["--enable-auto-tool-choice"])
with pytest.raises(TypeError):
@@ -160,7 +160,7 @@ def test_enable_auto_choice_passes_without_tool_call_parser(serve_parser):
def test_enable_auto_choice_passes_with_tool_call_parser(serve_parser):
"""Ensure validation passes with tool choice enabled with a call parser"""
"""Ensure validation passes with tool choice enabled with a call parser."""
args = serve_parser.parse_args(
args=[
"--enable-auto-tool-choice",
@@ -172,7 +172,7 @@ def test_enable_auto_choice_passes_with_tool_call_parser(serve_parser):
def test_enable_auto_choice_fails_with_enable_reasoning(serve_parser):
"""Ensure validation fails if reasoning is enabled with auto tool choice"""
"""Ensure validation fails if reasoning is enabled with auto tool choice."""
args = serve_parser.parse_args(
args=[
"--enable-auto-tool-choice",
@@ -186,7 +186,8 @@ def test_enable_auto_choice_fails_with_enable_reasoning(serve_parser):
def test_passes_with_reasoning_parser(serve_parser):
"""Ensure validation passes if reasoning is enabled
with a reasoning parser"""
with a reasoning parser
"""
args = serve_parser.parse_args(
args=[
"--reasoning-parser",
@@ -197,7 +198,7 @@ def test_passes_with_reasoning_parser(serve_parser):
def test_chat_template_validation_for_happy_paths(serve_parser):
"""Ensure validation passes if the chat template exists"""
"""Ensure validation passes if the chat template exists."""
args = serve_parser.parse_args(
args=["--chat-template", CHATML_JINJA_PATH.absolute().as_posix()]
)
@@ -205,7 +206,7 @@ def test_chat_template_validation_for_happy_paths(serve_parser):
def test_chat_template_validation_for_sad_paths(serve_parser):
"""Ensure validation fails if the chat template doesn't exist"""
"""Ensure validation fails if the chat template doesn't exist."""
args = serve_parser.parse_args(args=["--chat-template", "does/not/exist"])
with pytest.raises(VLLMValidationError):
validate_parsed_serve_args(args)
@@ -304,13 +305,13 @@ def test_sse_keep_alive_interval_non_integer(serve_parser, value):
],
)
def test_middleware(serve_parser, cli_args, expected_middleware):
"""Ensure multiple middleware args are parsed properly"""
"""Ensure multiple middleware args are parsed properly."""
args = serve_parser.parse_args(args=cli_args)
assert args.middleware == expected_middleware
def test_default_chat_template_kwargs_parsing(serve_parser):
"""Ensure default_chat_template_kwargs JSON is parsed correctly"""
"""Ensure default_chat_template_kwargs JSON is parsed correctly."""
args = serve_parser.parse_args(
args=["--default-chat-template-kwargs", '{"enable_thinking": false}']
)
@@ -318,7 +319,7 @@ def test_default_chat_template_kwargs_parsing(serve_parser):
def test_default_chat_template_kwargs_complex(serve_parser):
"""Ensure complex default_chat_template_kwargs JSON is parsed correctly"""
"""Ensure complex default_chat_template_kwargs JSON is parsed correctly."""
kwargs_json = '{"enable_thinking": false, "custom_param": "value", "num": 42}'
args = serve_parser.parse_args(args=["--default-chat-template-kwargs", kwargs_json])
assert args.default_chat_template_kwargs == {
@@ -329,7 +330,7 @@ def test_default_chat_template_kwargs_complex(serve_parser):
def test_default_chat_template_kwargs_default_none(serve_parser):
"""Ensure default_chat_template_kwargs defaults to None"""
"""Ensure default_chat_template_kwargs defaults to None."""
args = serve_parser.parse_args(args=[])
assert args.default_chat_template_kwargs is None
@@ -355,7 +356,7 @@ def test_resolve_default_chat_template_kwargs(default_kwargs, cohere_format, exp
def test_default_chat_template_kwargs_invalid_json(serve_parser):
"""Ensure invalid JSON raises an error"""
"""Ensure invalid JSON raises an error."""
with pytest.raises(SystemExit):
serve_parser.parse_args(
args=["--default-chat-template-kwargs", "not valid json"]
@@ -381,7 +382,8 @@ def test_default_chat_template_kwargs_invalid_json(serve_parser):
)
def test_served_model_name_parsing(tmp_path, vllm_parser, args, raises):
"""Ensure that users don't misuse --served-model-name and end up with the default
model tag instead of the one they intended to serve."""
model tag instead of the one they intended to serve.
"""
# Call the serve subparser
args.insert(0, "serve")
# Create a dummy config file if the test case includes it
@@ -402,7 +404,7 @@ def test_served_model_name_parsing(tmp_path, vllm_parser, args, raises):
### Tests for LoRA target modules parsing
def test_lora_target_modules_single(serve_parser):
"""Test parsing single lora-target-modules argument"""
"""Test parsing single lora-target-modules argument."""
args = serve_parser.parse_args(
args=["--enable-lora", "--lora-target-modules", "o_proj"]
)
@@ -410,7 +412,7 @@ def test_lora_target_modules_single(serve_parser):
def test_lora_target_modules_multiple(serve_parser):
"""Test parsing multiple lora-target-modules arguments"""
"""Test parsing multiple lora-target-modules arguments."""
args = serve_parser.parse_args(
args=[
"--enable-lora",
@@ -424,6 +426,6 @@ def test_lora_target_modules_multiple(serve_parser):
def test_lora_target_modules_default_none(serve_parser):
"""Test that lora-target-modules defaults to None"""
"""Test that lora-target-modules defaults to None."""
args = serve_parser.parse_args(args=[])
assert args.lora_target_modules is None
@@ -1,8 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Tests for DPSupervisor: unit tests and lifecycle integration tests.
"""Tests for DPSupervisor: unit tests and lifecycle integration tests.
Lifecycle integration tests replace child vLLM servers with lightweight
aiohttp "fake" servers controlled by the test, so the suite runs without GPUs.
@@ -463,8 +462,7 @@ async def test_start_server_no_log_config_when_no_filter(monkeypatch):
class MockVLLMServer:
"""
Minimal FastAPI server that mimics one vLLM replica.
"""Minimal FastAPI server that mimics one vLLM replica.
GET /health returns 200 when healthy, 503 otherwise.
Health state is toggled by the test via set_healthy().
"""
@@ -582,8 +580,7 @@ def launch_mock_vllm_with_drain(
async def _poll_supervisor_health(expected_status: int, use_ssl: bool = False) -> bool:
"""
GET /health on the supervisor once and check for expected_status.
"""GET /health on the supervisor once and check for expected_status.
Pass expected_status=-1 to assert the supervisor is not listening yet
(a connection error is expected). The supervisor only starts its HTTP
@@ -705,8 +702,7 @@ async def _run_supervisor(
@pytest.mark.asyncio
async def test_basic_lifecycle(monkeypatch):
"""
A) Supervisor is not listening while children are unhealthy.
"""A) Supervisor is not listening while children are unhealthy.
B) /health returns 200 once every child reports healthy.
C) SIGTERM and shutdown
"""
@@ -777,8 +773,7 @@ async def test_basic_lifecycle_with_ssl(monkeypatch):
@pytest.mark.asyncio
async def test_failed_startup(monkeypatch):
"""
A) One of the vLLM servers crashes during startup.
"""A) One of the vLLM servers crashes during startup.
B) DPSupervisor detects this, and cleans up resources.
"""
args = _make_args()
@@ -801,8 +796,7 @@ async def test_failed_startup(monkeypatch):
@pytest.mark.asyncio
async def test_becomes_unhealthy(monkeypatch):
"""
A) Supervisor is not listening while children are unhealthy.
"""A) Supervisor is not listening while children are unhealthy.
B) /health returns 200 once every child reports healthy.
C) Child process becomes unhealthy.
D) Detected and shutdown.
@@ -842,8 +836,7 @@ async def test_becomes_unhealthy(monkeypatch):
@pytest.mark.asyncio
async def test_dp_server_fails(monkeypatch):
"""
A) Supervisor is not listening while children are unhealthy.
"""A) Supervisor is not listening while children are unhealthy.
B) /health returns 200 once every child reports healthy.
C) Child process fails.
D) Detected and shutdown.
@@ -886,8 +879,7 @@ async def test_dp_server_fails(monkeypatch):
@pytest.mark.asyncio
async def test_shutdown_timeout(monkeypatch: pytest.MonkeyPatch):
"""
Child mock servers delay shutdown by 10s on SIGTERM (simulating in-flight
"""Child mock servers delay shutdown by 10s on SIGTERM (simulating in-flight
request drain). The supervisor is configured with shutdown_timeout=10,
so its total wait budget is 10 + CHILD_EXIT_GRACE_S seconds. The
children exit naturally within that window, so no force-kill should occur
@@ -509,9 +509,7 @@ def test_completions():
def test_completions_invalid_input():
"""
Ensure that we fail when the input doesn't conform to the openai api.
"""
"""Ensure that we fail when the input doesn't conform to the openai api."""
with (
tempfile.NamedTemporaryFile("w") as input_file,
tempfile.NamedTemporaryFile("r") as output_file,
@@ -602,9 +600,7 @@ def test_score(input_batch):
def test_reasoning_parser():
"""
Test that reasoning_parser parameter works correctly in run_batch.
"""
"""Test that reasoning_parser parameter works correctly in run_batch."""
with (
tempfile.NamedTemporaryFile("w") as input_file,
tempfile.NamedTemporaryFile("r") as output_file,
@@ -767,8 +763,7 @@ def test_translation():
def test_tool_calling():
"""
Test that tool calling works correctly in run_batch.
"""Test that tool calling works correctly in run_batch.
Verifies that requests with tools return tool_calls in the response.
"""
with (
@@ -134,7 +134,6 @@ async def test_shutdown_on_engine_failure(tmp_path: Path):
Starts a vLLM server, kills it to simulate a crash, then verifies that
subsequent API calls fail appropriately.
"""
port = get_open_port()
# Redirect server output to a file rather than a pipe: nothing drains the
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for HF_HUB_OFFLINE mode"""
"""Tests for HF_HUB_OFFLINE mode."""
import importlib
import sys
+1 -5
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
This file test accuracy of the vLLM server via LMEval.
"""This file test accuracy of the vLLM server via LMEval.
It uses local-completions, which interacts with vLLM
through the OAI API with N concurrent connections.
This simulates real work usage of the API and makes
@@ -33,7 +32,6 @@ EXPECTED_VALUES = {
def run_test(model_name, more_args=None):
"""Run the end to end accuracy test."""
model_args = f"pretrained={model_name},max_model_len=4096"
if more_args is not None:
@@ -64,7 +62,6 @@ TPU_TP_TEST_STR = "" # "tensor_parallel_size=4"
@pytest.mark.parametrize("model", MODEL_NAMES)
def test_lm_eval_accuracy_v1_engine(model):
"""Run with the V1 Engine."""
more_args = None
if current_platform.is_tpu():
# Limit compilation time for TPU V1
@@ -81,7 +78,6 @@ def test_lm_eval_accuracy_v1_engine(model):
@pytest.mark.parametrize("model", FP8_KV_MODEL_NAMES)
def test_lm_eval_accuracy_v1_engine_fp8_kv_cache(model):
"""Run with the V1 Engine."""
more_args = None
if current_platform.is_tpu():
# Limit compilation time for TPU V1
+3 -6
View File
@@ -20,8 +20,7 @@ def text_llm(vllm_runner):
@pytest.fixture(scope="function")
def llm_for_failure_test(vllm_runner):
"""
Fixture for testing issue #26081.
"""Fixture for testing issue #26081.
Uses a small max_model_len to easily trigger length errors.
"""
with vllm_runner(
@@ -67,8 +66,7 @@ def test_multi_chat(text_llm):
def test_llm_chat_tokenization_no_double_bos(text_llm):
"""
LLM.chat() should not add special tokens when using chat templates.
"""LLM.chat() should not add special tokens when using chat templates.
Check we get a single BOS token for llama chat.
"""
messages = [
@@ -127,8 +125,7 @@ def test_chat_extra_kwargs(thinking_llm, enable_thinking):
def test_chat_batch_failure_cleanup(llm_for_failure_test):
"""
Tests that if a batch call to llm.chat() fails mid-way
"""Tests that if a batch call to llm.chat() fails mid-way
(e.g., due to one invalid prompt), the requests that
were already enqueued are properly aborted and do not
pollute the queue for subsequent calls.
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Test that ``InputProcessor.inject_into_mm_cache()`` correctly injects
"""Test that ``InputProcessor.inject_into_mm_cache()`` correctly injects
pre-processed mm_kwargs into the processor cache and reports MM cache
hit rate metrics accurately.
@@ -47,8 +47,7 @@ async def client(server):
async def test_online_audio_in_video(
client: openai.AsyncOpenAI, video_assets: VideoTestAssets
):
"""Test video input with `audio_in_video=True`"""
"""Test video input with `audio_in_video=True`."""
# we don't use video_urls above because they missed audio stream.
video_path = video_assets[0].video_path
with open(video_path, "rb") as f:
@@ -97,8 +96,7 @@ async def test_online_audio_in_video(
async def test_online_audio_in_video_multi_videos(
client: openai.AsyncOpenAI, video_assets: VideoTestAssets
):
"""Test multi-video input with `audio_in_video=True`"""
"""Test multi-video input with `audio_in_video=True`."""
# we don't use video_urls above because they missed audio stream.
video_path = video_assets[0].video_path
with open(video_path, "rb") as f:
@@ -151,8 +149,7 @@ async def test_online_audio_in_video_multi_videos(
async def test_online_audio_in_video_interleaved(
client: openai.AsyncOpenAI, video_assets: VideoTestAssets
):
"""Test interleaved video/audio input with `audio_in_video=True`"""
"""Test interleaved video/audio input with `audio_in_video=True`."""
# we don't use video_urls above because they missed audio stream.
video_path = video_assets[0].video_path
with open(video_path, "rb") as f:
@@ -2,7 +2,8 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""E2E test for mixing `prompt_embeds` with `audio_embeds` in a single
Chat Completions request."""
Chat Completions request.
"""
import json
@@ -68,7 +68,8 @@ async def client(server):
def image_url() -> str:
"""Stable real image as a data URL, kept identical across both the
text and prompt_embeds requests so any output difference must come from
how the text content is delivered."""
how the text content is delivered.
"""
return encode_image_url(ImageAsset("stop_sign").pil_image)
@@ -34,8 +34,7 @@ REQUIRED_BEAM_SEARCH_TERMS = [
def check_output_matches_terms(content: str, term_groups: list[list[str]]) -> bool:
"""
Check if content matches all required term groups.
"""Check if content matches all required term groups.
Each term group requires at least one of its terms to be present.
All term groups must be satisfied.
"""
@@ -146,7 +145,8 @@ def describe_image_messages(
) -> list[dict]:
"""Build the system + user messages used by the completions-with-image
family of tests. *extra_image_fields* is merged into the top-level
image content block (for uuid / bad-key tests)."""
image content block (for uuid / bad-key tests).
"""
image_block: dict = {
"type": "image_url",
"image_url": {"url": image_url},
@@ -176,7 +176,8 @@ async def complete_and_check(
temperature: float = 0.0,
) -> str:
"""Run a chat completion and assert the output is non-empty.
Returns the content string."""
Returns the content string.
"""
chat_completion = await client.chat.completions.create(
model=model_name,
messages=messages,
@@ -259,7 +259,7 @@ async def _generator(prompt_idx: int, text: str) -> AsyncGenerator[RequestOutput
@pytest.mark.asyncio
@pytest.mark.skip_global_cleanup
async def test_batched_echo_does_not_prepend_user_prompt() -> None:
"""echo must only echo the assistant turn, never the user's prompt.
"""Echo must only echo the assistant turn, never the user's prompt.
With `add_generation_prompt` (the default) the response role is `assistant`
while the last conversation message is the user's, so there is nothing to
@@ -121,7 +121,7 @@ def _build_serving_chat(engine: AsyncLLM) -> OpenAIServingChat:
@pytest.mark.asyncio
async def test_chat_error_non_stream():
"""test finish_reason='error' returns 500 InternalServerError (non-streaming)"""
"""Test finish_reason='error' returns 500 InternalServerError (non-streaming)."""
mock_engine = MagicMock(spec=AsyncLLM)
mock_engine.errored = False
mock_engine.model_config = MockModelConfig()
@@ -246,7 +246,7 @@ async def test_renderer_only_chat_request_skips_mm_cache():
@pytest.mark.asyncio
async def test_chat_error_stream():
"""test finish_reason='error' returns 500 InternalServerError (streaming)"""
"""Test finish_reason='error' returns 500 InternalServerError (streaming)."""
mock_engine = MagicMock(spec=AsyncLLM)
mock_engine.errored = False
mock_engine.model_config = MockModelConfig()
@@ -465,7 +465,8 @@ def test_system_message_warns_on_video(video_content):
def test_json_schema_response_format_missing_schema():
"""When response_format type is 'json_schema' but the json_schema field
is not provided, request construction should raise a validation error
so the API returns 400 instead of 500."""
so the API returns 400 instead of 500.
"""
with pytest.raises(Exception, match="json_schema.*must be provided"):
ChatCompletionRequest(
model=MODEL_NAME,
@@ -519,7 +520,8 @@ def test_structured_outputs_structural_tag_invalid(structural_tag):
@pytest.mark.parametrize("field_name", ["prompt_logprobs", "top_logprobs"])
def test_non_numeric_logprobs_rejected(field_name):
"""A non-numeric logprobs value must be a clean 400 validation error, not a
TypeError from the mode='before' comparison (which surfaces as HTTP 500)."""
TypeError from the mode='before' comparison (which surfaces as HTTP 500).
"""
with pytest.raises(VLLMValidationError, match=f"`{field_name}` must be an integer"):
ChatCompletionRequest(
model=MODEL_NAME,
@@ -81,7 +81,8 @@ async def test_chat_logit_bias_invalid(client):
@pytest.mark.asyncio
async def test_chat_logit_bias_non_integer_key(client):
"""Test that a non-integer logit_bias key is rejected with a clean,
informative error instead of a raw 'invalid literal for int()' message."""
informative error instead of a raw 'invalid literal for int()' message.
"""
with pytest.raises(openai.BadRequestError) as excinfo:
await client.chat.completions.create(
model=MODEL_NAME,
@@ -101,7 +102,8 @@ async def test_chat_logit_bias_non_integer_key(client):
@pytest.mark.asyncio
async def test_chat_logit_bias_non_numeric_value(client):
"""Test that a non-numeric logit_bias value is rejected with a message
that names the specific offending token, not just a generic TypeError."""
that names the specific offending token, not just a generic TypeError.
"""
with pytest.raises(openai.BadRequestError) as excinfo:
await client.chat.completions.create(
model=MODEL_NAME,
@@ -120,7 +122,8 @@ async def test_chat_logit_bias_non_numeric_value(client):
@pytest.mark.asyncio
async def test_chat_logit_bias_multiple_non_integer_keys(client):
"""Test that ALL invalid logit_bias keys are reported together,
not just the first one encountered."""
not just the first one encountered.
"""
with pytest.raises(openai.BadRequestError) as excinfo:
await client.chat.completions.create(
model=MODEL_NAME,
@@ -451,7 +451,7 @@ async def test_inconsistent_tool_choice_and_tools(
async def test_max_tokens_with_tool_choice_required(
client: openai.AsyncOpenAI, tool_choice
):
""" """
"""Requesting a single token with a required tool choice must not crash."""
models = await client.models.list()
model_name: str = models.data[0].id
@@ -52,9 +52,10 @@ def stop_sign_image_url():
@pytest.mark.asyncio
async def test_translategemma_extra_lang_code_fields(client: openai.AsyncOpenAI):
"""en -> es translation through TranslateGemma's bundled chat template's
"""En -> es translation through TranslateGemma's bundled chat template's
``text`` content branch, which depends on ``source_lang_code`` /
``target_lang_code`` extra fields being preserved on the content part."""
``target_lang_code`` extra fields being preserved on the content part.
"""
completion = await client.chat.completions.create(
model=MODEL_NAME,
messages=[
@@ -92,13 +93,14 @@ async def test_translategemma_image_extra_lang_code_fields(
client: openai.AsyncOpenAI,
stop_sign_image_url: str,
):
"""en -> es OCR-translation through TranslateGemma's bundled chat
"""En -> es OCR-translation through TranslateGemma's bundled chat
template's ``image`` content branch. Exercises the multimodal branch of
``_collect_extra_fields`` in ``_parse_chat_message_content_part``: the
``source_lang_code`` / ``target_lang_code`` extras must survive parsing
and end up alongside the ``{"type": "image"}`` placeholder so the
template's image branch (``content["type"] == 'image'``) renders the
right OCR-translation prompt."""
right OCR-translation prompt.
"""
completion = await client.chat.completions.create(
model=MODEL_NAME,
messages=[
@@ -191,7 +191,8 @@ async def test_logprob_token_ids_stream_with_no_top_k(server):
async def test_logprob_token_ids_default_behavior_unchanged(server):
"""Without `logprob_token_ids`, the response carries the natural top-k
most-likely tokens. This guards against the new field accidentally
changing the default-path output."""
changing the default-path output.
"""
async with server.get_async_client() as client:
resp = await client.chat.completions.create(
model=MODEL_NAME,
@@ -1232,7 +1232,6 @@ async def test_serving_chat_mistral_token_ids_prompt_is_validated():
"""Regression test: when the Mistral tokenizer path returns token IDs
directly, we must still apply input length + max_tokens validation.
"""
mock_engine = MagicMock(spec=AsyncLLM)
mock_engine.errored = False
mock_engine.model_config = MockModelConfig(skip_tokenizer_init=True)
@@ -1271,7 +1270,6 @@ async def test_serving_chat_mistral_token_ids_prompt_too_long_is_rejected():
"""Regression test: MistralTokenizer token-id prompts must still enforce
the max context length for the input itself (token_num >= max_model_len).
"""
mock_engine = MagicMock(spec=AsyncLLM)
mock_engine.errored = False
mock_engine.model_config = MockModelConfig(skip_tokenizer_init=True)
@@ -1408,7 +1406,8 @@ async def test_serving_chat_did_set_correct_cache_salt(model_type):
@pytest.mark.asyncio
async def test_serving_chat_data_parallel_rank_extraction():
"""Test that data_parallel_rank is properly extracted from header and
passed to engine."""
passed to engine.
"""
mock_engine = MagicMock(spec=AsyncLLM)
mock_engine.errored = False
mock_engine.model_config = MockModelConfig()
@@ -1512,8 +1511,7 @@ async def _render_chat_prompt_token_ids(
class TestServingChatWithHarmony:
"""
These tests ensure Chat Completion requests are being properly converted into
"""These tests ensure Chat Completion requests are being properly converted into
Harmony messages and Harmony response messages back into Chat Completion responses.
These tests are not exhaustive, but each one was created to cover a specific case
that we got wrong but is now fixed.
@@ -1731,7 +1729,8 @@ class TestServingChatWithHarmony:
@pytest.mark.asyncio
async def test_system_message_without_tools(self, serving_chat, stream):
"""Leading system message produces a developer message with
DeveloperContent (# Instructions header)."""
DeveloperContent (# Instructions header).
"""
messages = [
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": "Hello"},
@@ -1753,7 +1752,8 @@ class TestServingChatWithHarmony:
@pytest.mark.asyncio
async def test_system_message_with_tools(self, serving_chat, stream, weather_tools):
"""Leading system message is folded into the developer message
alongside tool definitions."""
alongside tool definitions.
"""
messages = [
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": "What's the weather?"},
@@ -2168,7 +2168,8 @@ class TestServingChatWithHarmony:
@pytest.mark.asyncio
async def test_tool_choice_validation_without_parser():
"""Test that tool_choice='required' or named tool without tool_parser
returns an appropriate error message."""
returns an appropriate error message.
"""
mock_engine = MagicMock(spec=AsyncLLM)
mock_engine.errored = False
mock_engine.model_config = MockModelConfig()
@@ -33,7 +33,6 @@ def _count_reasoning_decode_token_ids_between_markers(
reasoning_end_ids: list[int],
) -> int | None:
"""Count decode tokens in the thinking span (after last start, before first end)."""
if not reasoning_start_ids or not reasoning_end_ids:
raise ValueError("reasoning marker token id lists must be non-empty")
@@ -152,8 +151,8 @@ async def client(request, server, server_with_auto_reasoning_config):
@pytest.mark.parametrize("client", ["default", "auto_config"], indirect=True)
async def test_thinking_token_budget_mixed_requests(client: openai.AsyncOpenAI):
"""Test that mixed requests (some with thinking_token_budget, some without)
complete successfully without errors."""
complete successfully without errors.
"""
response_with_budget = await client.chat.completions.create(
model=MODEL_NAME,
messages=MESSAGES,
@@ -182,7 +181,6 @@ async def test_thinking_token_budget_limits_reasoning(client: openai.AsyncOpenAI
grouped into streamed chunks (a single chunk can carry several tokens under
async scheduling / stream_interval > 1). Counting chunks under-counts.
"""
tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
start_ids = list(tokenizer.encode(REASONING_START_STR, add_special_tokens=False))
end_ids = list(tokenizer.encode(REASONING_END_STR, add_special_tokens=False))
@@ -227,7 +225,6 @@ async def test_thinking_token_budget_qwen35_fp8_mtp_concurrent_mixed_budget_and_
Qwen3.5 FP8 + MTP (TP=2) server. Budgeted calls are checked with
``_count_reasoning_decode_token_ids_between_markers`` on full token ids.
"""
_batch_spec: list[tuple[Literal["budget"], int] | tuple[Literal["plain"], None]] = [
("budget", 1),
("budget", 12),
@@ -350,7 +350,6 @@ async def test_parallel_no_streaming(client: openai.AsyncOpenAI, model_name: str
"""Parallel sampling without streaming.
A single request output contains a list of completions.
"""
prompt = "What is an LLM?"
n = 3
max_tokens = 50
@@ -396,7 +395,6 @@ async def test_parallel_streaming(client: openai.AsyncOpenAI, model_name: str):
The tokens from multiple samples, are flattened into a single stream,
with an index to indicate which sample the token belongs to.
"""
prompt = "What is an LLM?"
n = 3
max_tokens = 50
@@ -303,7 +303,7 @@ def test_completion_metrics_carries_both_timing_and_spec_decode():
@pytest.mark.asyncio
async def test_completion_error_non_stream():
"""test finish_reason='error' returns 500 InternalServerError (non-streaming)"""
"""Test finish_reason='error' returns 500 InternalServerError (non-streaming)."""
mock_engine = MagicMock(spec=AsyncLLM)
mock_engine.errored = False
mock_engine.model_config = MockModelConfig()
@@ -439,7 +439,7 @@ async def test_renderer_only_completion_request_skips_mm_cache():
@pytest.mark.asyncio
async def test_completion_error_stream():
"""test finish_reason='error' returns 500 InternalServerError (streaming)"""
"""Test finish_reason='error' returns 500 InternalServerError (streaming)."""
mock_engine = MagicMock(spec=AsyncLLM)
mock_engine.errored = False
mock_engine.model_config = MockModelConfig()
@@ -521,7 +521,8 @@ async def test_completion_error_stream():
def test_json_schema_response_format_missing_schema():
"""When response_format type is 'json_schema' but the json_schema field
is not provided, request construction should raise a validation error
so the API returns 400 instead of 500."""
so the API returns 400 instead of 500.
"""
with pytest.raises(Exception, match="json_schema.*must be provided"):
CompletionRequest(
model=MODEL_NAME,
@@ -584,7 +585,8 @@ def test_negative_prompt_token_ids_flat():
def test_logprobs_minus_one_allowed():
"""logprobs=-1 means "return all logprobs". The sampling layer and the chat
top_logprobs / prompt_logprobs validators all accept -1, so the completion
logprobs validator must accept it too instead of rejecting it as negative."""
logprobs validator must accept it too instead of rejecting it as negative.
"""
request = CompletionRequest(
model=MODEL_NAME,
prompt="Test prompt",
@@ -744,7 +746,8 @@ class TestCompletionPromptListLimit:
@pytest.mark.parametrize("field_name", ["prompt_logprobs", "logprobs"])
def test_non_numeric_logprobs_rejected(field_name):
"""A non-numeric logprobs value must be a clean 400 validation error, not a
TypeError from the mode='before' comparison (which surfaces as HTTP 500)."""
TypeError from the mode='before' comparison (which surfaces as HTTP 500).
"""
with pytest.raises(VLLMValidationError, match=f"`{field_name}` must be an integer"):
CompletionRequest(
model=MODEL_NAME,
@@ -39,8 +39,7 @@ def server():
@pytest.mark.asyncio
async def test_token_in_token_out_and_logprobs(server):
"""
Test token-in-token-out and token_ids align with prompt_logprobs
"""Test token-in-token-out and token_ids align with prompt_logprobs
& logprobs when return_tokens_as_token_ids is enabled.
"""
tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
This file test accuracy of the vLLM server via LMEval.
"""This file test accuracy of the vLLM server via LMEval.
It uses local-completions, which interacts with vLLM
through the OAI API with N concurrent connections.
This simulates real work usage of the API and makes
@@ -37,7 +36,6 @@ if current_platform.is_tpu():
def run_test(more_args):
"""Run the end to end accuracy test."""
args = list(DEFAULT_ARGS)
args.extend(more_args)
print(f"Running with: {args}")
@@ -68,7 +66,6 @@ def run_test(more_args):
def test_lm_eval_accuracy_v1_engine():
"""Run with the V1 Engine."""
more_args = []
# Limit compilation time for V1 on TPU
@@ -43,7 +43,8 @@ def _system():
class TestResponseInputToHarmonyRenderParity:
"""Each test drives the same conversation through both APIs and asserts
identical Harmony messages and rendered token sequences."""
identical Harmony messages and rendered token sequences.
"""
# -----------------------------------------------------------------------
# Single-message cases
@@ -51,7 +52,8 @@ class TestResponseInputToHarmonyRenderParity:
def test_developer_message(self):
"""Both APIs must render developer messages identically using
DeveloperContent (with the '# Instructions' header)."""
DeveloperContent (with the '# Instructions' header).
"""
chat_msgs = parse_chat_input_to_harmony_message(
{"role": "developer", "content": "Be concise."}
)
@@ -246,7 +248,8 @@ class TestResponseInputToHarmonyRenderParity:
def test_reasoning_combined_with_function_call(self):
"""Chat API packs reasoning + tool_calls into one dict; responses API
represents them as two separate items. Both must produce the same two
Harmony messages in the same order: analysis then commentary."""
Harmony messages in the same order: analysis then commentary.
"""
chat_msgs = parse_chat_input_to_harmony_message(
{
"role": "assistant",
@@ -529,7 +532,8 @@ class TestResponseInputToHarmonyRenderParity:
def test_completed_turns_drop_reasoning(self):
"""Validates that reasoning from completed turns is dropped, while
reasoning from the current in-progress tool-call turn is preserved
in both chat completions and responses previous_input_messages."""
in both chat completions and responses previous_input_messages.
"""
first_turn_reasoning = "FIRST_TURN_REASONING"
second_turn_reasoning = "SECOND_TURN_REASONING"
@@ -170,7 +170,7 @@ class TestIsFunctionRecipientWithAllowedNames:
"""Tests for is_function_recipient with allowed_function_tool_names."""
def test_prefixed_always_accepted(self):
"""functions. prefix is always accepted regardless of allowed names."""
"""Functions. prefix is always accepted regardless of allowed names."""
fn_names = frozenset({"other_tool"})
assert is_function_recipient("functions.get_weather", fn_names) is True
@@ -252,8 +252,7 @@ class TestExtractFunctionFromRecipient:
class TestCommonParseInputToHarmonyMessage:
"""
Tests for scenarios that are common to both Chat Completion
"""Tests for scenarios that are common to both Chat Completion
parse_chat_input_to_harmony_message and Responses API
response_previous_input_to_harmony functions.
"""
@@ -324,7 +323,8 @@ class TestCommonParseInputToHarmonyMessage:
def test_system_message(self, parse_function):
"""Test parsing system messages, which are parsed into developer messages
with DeveloperContent."""
with DeveloperContent.
"""
chat_msg = {
"role": "system",
"content": "You are a helpful assistant",
@@ -448,8 +448,7 @@ class TestCommonParseInputToHarmonyMessage:
class TestParseChatInputToHarmonyMessage:
"""
Tests for scenarios that are specific to the Chat Completion API
"""Tests for scenarios that are specific to the Chat Completion API
parse_chat_input_to_harmony_message function.
"""
@@ -14,7 +14,7 @@ from vllm.exceptions import GenerationError
@pytest.mark.asyncio
async def test_raise_if_error_raises_generation_error():
"""test _raise_if_error raises GenerationError"""
"""Test _raise_if_error raises GenerationError."""
# create a minimal GenerateBaseServing instance
mock_engine = MagicMock()
mock_engine.model_config = MagicMock()
@@ -42,7 +42,7 @@ async def test_raise_if_error_raises_generation_error():
@pytest.mark.asyncio
async def test_convert_generation_error_to_streaming_response():
"""test _convert_generation_error_to_streaming_response output"""
"""Test _convert_generation_error_to_streaming_response output."""
mock_engine = MagicMock()
mock_engine.model_config = MagicMock()
mock_engine.model_config.max_model_len = 100

Some files were not shown because too many files have changed in this diff Show More