Add pydocstyle to the ruff rules (#52136)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: Harry Mellor <hej.mellor@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
804e5377aa
commit
8c1557a79c
@@ -89,7 +89,7 @@ TARGET_TEST_SUITE = _resolve_target_test_suite()
|
||||
condition=current_platform.is_rocm(),
|
||||
)
|
||||
def test_vllm_gc_ed():
|
||||
"""Verify vllm instance is GC'ed when it is deleted"""
|
||||
"""Verify vllm instance is GC'ed when it is deleted."""
|
||||
llm = LLM("hmellor/tiny-random-LlamaForCausalLM")
|
||||
weak_llm = weakref.ref(llm)
|
||||
del llm
|
||||
|
||||
@@ -51,8 +51,7 @@ def _wake_up_with_poisoned_mappings(allocator, byte_value: int = 0xA5) -> None:
|
||||
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
def test_python_error():
|
||||
"""
|
||||
Test if Python error occurs when there's low-level
|
||||
"""Test if Python error occurs when there's low-level
|
||||
error happening from the C++ side.
|
||||
"""
|
||||
allocator = get_mem_allocator_instance()
|
||||
@@ -113,7 +112,8 @@ def test_basic_cumem():
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
def test_discard_tags():
|
||||
"""Test that discard(tags) selectively frees GPU memory for specific
|
||||
tags while keeping other tags mapped and usable."""
|
||||
tags while keeping other tags mapped and usable.
|
||||
"""
|
||||
allocator = get_mem_allocator_instance()
|
||||
|
||||
with allocator.use_memory_pool("weights"):
|
||||
|
||||
@@ -13,7 +13,8 @@ from vllm.benchmarks.datasets import BFCLDataset, get_samples
|
||||
|
||||
def _patch_hf_api(side_effect):
|
||||
"""Return a patch context that swaps `hf_api()` to a stub whose
|
||||
`.hf_hub_download` attribute uses `side_effect`."""
|
||||
`.hf_hub_download` attribute uses `side_effect`.
|
||||
"""
|
||||
fake_api = MagicMock()
|
||||
fake_api.hf_hub_download.side_effect = side_effect
|
||||
return patch("vllm.benchmarks.datasets.datasets.hf_api", return_value=fake_api)
|
||||
@@ -115,7 +116,8 @@ def test_bfcl_dataset_translates_schema_and_attaches_tools(
|
||||
) -> None:
|
||||
"""BFCLDataset should translate schemas to OpenAI tool format, set
|
||||
`messages` directly on SampleRequest, and attach tools/tool_choice via
|
||||
request_overrides."""
|
||||
request_overrides.
|
||||
"""
|
||||
paths = _write_fake_files(tmp_path)
|
||||
|
||||
def fake_download(_repo, filename, **_kwargs):
|
||||
@@ -166,7 +168,8 @@ def test_bfcl_dataset_missing_category_raises_clear_error(
|
||||
hf_tokenizer: PreTrainedTokenizerBase,
|
||||
) -> None:
|
||||
"""A typo'd category should produce an actionable ValueError, not an
|
||||
opaque huggingface_hub exception."""
|
||||
opaque huggingface_hub exception.
|
||||
"""
|
||||
from huggingface_hub.errors import EntryNotFoundError
|
||||
|
||||
args = _args_for_bfcl(categories=["simpl"]) # typo
|
||||
@@ -184,7 +187,8 @@ def test_bfcl_dataset_missing_category_raises_clear_error(
|
||||
@pytest.mark.benchmark
|
||||
def test_chat_backend_uses_messages_field_when_set() -> None:
|
||||
"""When RequestFuncInput.chat_messages is set, the chat backend must use
|
||||
it verbatim and skip default content construction from `prompt`."""
|
||||
it verbatim and skip default content construction from `prompt`.
|
||||
"""
|
||||
import asyncio
|
||||
|
||||
from vllm.benchmarks.lib.endpoint_request_func import (
|
||||
@@ -244,7 +248,8 @@ def test_chat_backend_uses_messages_field_when_set() -> None:
|
||||
def test_bfcl_prompt_len_includes_tools(tmp_path: Path) -> None:
|
||||
"""prompt_len must reflect tokens from both messages *and* tool schemas,
|
||||
so percentile buckets and input-distribution summaries aren't biased
|
||||
low for tool-heavy traffic."""
|
||||
low for tool-heavy traffic.
|
||||
"""
|
||||
paths = _write_fake_files(tmp_path)
|
||||
|
||||
def fake_download(_repo, filename, **_kwargs):
|
||||
@@ -291,7 +296,8 @@ def test_bfcl_prompt_len_falls_back_when_tokenizer_rejects_tools(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""Older tokenizers don't accept tools=; fallback must still produce a
|
||||
non-zero prompt_len without crashing."""
|
||||
non-zero prompt_len without crashing.
|
||||
"""
|
||||
paths = _write_fake_files(tmp_path)
|
||||
|
||||
def fake_download(_repo, filename, **_kwargs):
|
||||
|
||||
@@ -12,7 +12,8 @@ from vllm.benchmarks.datasets import get_samples
|
||||
class _RecordingTokenizer:
|
||||
"""Minimal tokenizer stub that records the kwargs forwarded to
|
||||
apply_chat_template, so we can assert chat_template_kwargs propagation
|
||||
without loading a real model/template."""
|
||||
without loading a real model/template.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.captured_kwargs: dict | None = None
|
||||
|
||||
@@ -228,7 +228,7 @@ def test_decode_span_identity_is_exact(
|
||||
api_url: str,
|
||||
chat: bool,
|
||||
) -> None:
|
||||
"""latency - ttft must equal sum(itl) exactly.
|
||||
"""Latency - ttft must equal sum(itl) exactly.
|
||||
|
||||
On a scripted clock this can only drift if a code path reads the clock an
|
||||
extra time or attributes a read to the wrong event.
|
||||
|
||||
@@ -10,7 +10,8 @@ from vllm.tokenizers import TokenizerLike
|
||||
|
||||
class _FakeTokenizer(TokenizerLike):
|
||||
"""Minimal tokenizer implementing the TokenizerLike protocol
|
||||
for testing get_sampling_params."""
|
||||
for testing get_sampling_params.
|
||||
"""
|
||||
|
||||
def __init__(self, vocab_size: int = 1000, num_special_tokens: int = 0) -> None:
|
||||
self._vocab_size = vocab_size
|
||||
@@ -171,7 +172,8 @@ class TestGetSamplingParams:
|
||||
|
||||
def test_output_low_clamped_to_one(self):
|
||||
"""Even with a high ratio that would push output_low to 0,
|
||||
the function clamps it to 1."""
|
||||
the function clamps it to 1.
|
||||
"""
|
||||
rng = np.random.default_rng(0)
|
||||
# output_len=1, ratio=0.99 → floor(1*0.01)=0, should clamp to 1
|
||||
_, output_lens, _ = get_sampling_params(
|
||||
@@ -233,7 +235,8 @@ class TestGetSamplingParams:
|
||||
|
||||
def test_input_len_zero_with_special_tokens(self):
|
||||
"""input_len < num_special_tokens → real_input_len = 0, which is fine
|
||||
(range [0, 0])."""
|
||||
(range [0, 0]).
|
||||
"""
|
||||
rng = np.random.default_rng(0)
|
||||
input_lens, _, _ = get_sampling_params(
|
||||
rng, 5, 0.0, 5, 50, self._tok(num_special=10)
|
||||
|
||||
@@ -52,7 +52,8 @@ def _write_dataset(path: Path) -> None:
|
||||
|
||||
def _args(dataset_path: str) -> argparse.Namespace:
|
||||
"""Reproduce the argparse.Namespace that serve.py builds from the
|
||||
failing command, including skip_tokenizer_init=True."""
|
||||
failing command, including skip_tokenizer_init=True.
|
||||
"""
|
||||
return argparse.Namespace(
|
||||
# dataset
|
||||
dataset_name="custom",
|
||||
|
||||
+1
-3
@@ -1,8 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
These envs only work for a small part of the tests, fix what you need!
|
||||
"""
|
||||
"""These envs only work for a small part of the tests, fix what you need!"""
|
||||
|
||||
import os
|
||||
from collections.abc import Callable
|
||||
|
||||
@@ -29,8 +29,7 @@ logger = init_logger("vllm.tests.compile.backend")
|
||||
|
||||
|
||||
class LazyInitPass(InductorPass):
|
||||
"""
|
||||
If there's a pass that we want to initialize lazily in a test,
|
||||
"""If there's a pass that we want to initialize lazily in a test,
|
||||
we can wrap it in LazyInitPass, which will initialize the pass when invoked
|
||||
and then immediately invoke it.
|
||||
"""
|
||||
@@ -45,8 +44,7 @@ class LazyInitPass(InductorPass):
|
||||
|
||||
|
||||
class TestBackend:
|
||||
"""
|
||||
This class provides a simple Inductor backend that can be used for testing.
|
||||
"""This class provides a simple Inductor backend that can be used for testing.
|
||||
It takes a list of custom passes and runs them after Inductor's passes.
|
||||
It also saves the graph before and after the custom passes for inspection.
|
||||
|
||||
|
||||
@@ -11,8 +11,7 @@ from vllm.platforms.interface import DeviceCapability
|
||||
|
||||
@pytest.fixture
|
||||
def mock_cuda_platform():
|
||||
"""
|
||||
Fixture that returns a factory for creating mocked CUDA platforms.
|
||||
"""Fixture that returns a factory for creating mocked CUDA platforms.
|
||||
|
||||
Usage:
|
||||
def test_something(mock_cuda_platform):
|
||||
@@ -51,8 +50,7 @@ def mock_cuda_platform():
|
||||
|
||||
@pytest.fixture
|
||||
def mock_xpu_platform():
|
||||
"""
|
||||
Fixture that returns a factory for creating mocked XPU platforms.
|
||||
"""Fixture that returns a factory for creating mocked XPU platforms.
|
||||
|
||||
Usage:
|
||||
def test_something(mock_xpu_platform):
|
||||
|
||||
@@ -1,11 +1,10 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
WARNING: This test runs in both single-node (4 GPUs) and multi-node
|
||||
(2 node with 2 GPUs each) modes. If the test only uses 2 GPUs, it is
|
||||
important to set the distributed backend to "mp" to avoid Ray scheduling
|
||||
all workers in a node other than the head node, which can cause the test
|
||||
to fail.
|
||||
"""WARNING: This test runs in both single-node (4 GPUs) and multi-node
|
||||
(2 node with 2 GPUs each) modes. If the test only uses 2 GPUs, it is
|
||||
important to set the distributed backend to "mp" to avoid Ray scheduling
|
||||
all workers in a node other than the head node, which can cause the test
|
||||
to fail.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
@@ -17,8 +17,7 @@ from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
@contextlib.contextmanager
|
||||
def temporary_environ(env_vars):
|
||||
"""
|
||||
Temporarily set environment variables and restore them afterward.
|
||||
"""Temporarily set environment variables and restore them afterward.
|
||||
We have to do this vs monkeypatch because monkeypatch doesn't work
|
||||
with "module" scoped fixtures.
|
||||
"""
|
||||
@@ -125,8 +124,7 @@ def llm_pair(request):
|
||||
indirect=True,
|
||||
)
|
||||
class TestFullCUDAGraph:
|
||||
"""
|
||||
Use a class such that an llm pair is constructed once for all
|
||||
"""Use a class such that an llm pair is constructed once for all
|
||||
batch_size/max_tokens combinations and released immediately after.
|
||||
|
||||
Module-scope fixtures would stick around the whole time,
|
||||
@@ -149,11 +147,9 @@ class TestFullCUDAGraph:
|
||||
],
|
||||
)
|
||||
def test_full_cudagraph(self, batch_size, max_tokens, llm_pair: tuple[LLM, LLM]):
|
||||
"""
|
||||
Test various batch sizes and max_tokens to ensure that the
|
||||
"""Test various batch sizes and max_tokens to ensure that the
|
||||
full cudagraph compilation works for padded cases too.
|
||||
"""
|
||||
|
||||
full_cudagraph_llm, piecewise_llm = llm_pair
|
||||
|
||||
prompts = ["the quick brown fox"] * batch_size
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Test (piecewise) compilation with a simple model where multiple submodules
|
||||
"""Test (piecewise) compilation with a simple model where multiple submodules
|
||||
are compiled and graph captured separately.
|
||||
"""
|
||||
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Test the piecewise compilation with a simple model so that we
|
||||
"""Test the piecewise compilation with a simple model so that we
|
||||
can exactly calculate the expected output and side effects.
|
||||
"""
|
||||
|
||||
@@ -52,8 +51,7 @@ class SillyModel(nn.Module):
|
||||
self.intermediate_unbacked = intermediate_unbacked
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
"""
|
||||
Overall effect:
|
||||
"""Overall effect:
|
||||
x = 3 * x + 19
|
||||
global_counter += 2
|
||||
"""
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Test the piecewise compilation with a simple model, comparing the output
|
||||
"""Test the piecewise compilation with a simple model, comparing the output
|
||||
with and without the piecewise compilation.
|
||||
|
||||
This is a tractable model, the weights and computation are specially designed
|
||||
|
||||
@@ -56,10 +56,7 @@ class TestMMRSModel(torch.nn.Module):
|
||||
torch.nn.init.normal_(self.gate_proj, std=0.02)
|
||||
|
||||
def forward(self, hidden_states):
|
||||
"""
|
||||
Forward pass implementing the mm + reduce scatter in the FX graph
|
||||
|
||||
"""
|
||||
"""Forward pass implementing the mm + reduce scatter in the FX graph."""
|
||||
# Reshape input
|
||||
view = hidden_states.reshape(-1, self.hidden_size)
|
||||
|
||||
@@ -88,9 +85,7 @@ class TestAGMMModel(torch.nn.Module):
|
||||
torch.nn.init.normal_(self.weight, std=0.02)
|
||||
|
||||
def forward(self, hidden_states):
|
||||
"""
|
||||
Forward pass implementing the mm + all gather in the FX graph
|
||||
"""
|
||||
"""Forward pass implementing the mm + all gather in the FX graph."""
|
||||
# Reshape input
|
||||
view = hidden_states.reshape(-1, self.hidden_size)
|
||||
all_gather = tensor_model_parallel_all_gather(view, dim=0)
|
||||
@@ -122,10 +117,7 @@ class _BaseScaledMMModel(torch.nn.Module):
|
||||
|
||||
class TestScaledMMRSModel(_BaseScaledMMModel):
|
||||
def forward(self, input: torch.Tensor):
|
||||
"""
|
||||
Forward pass implementing the scaled_mm + reduce scatter in the FX graph
|
||||
|
||||
"""
|
||||
"""Forward pass implementing the scaled_mm + reduce scatter in the FX graph."""
|
||||
fp8_input = input.to(FP8_DTYPE)
|
||||
scale_a = torch.ones(input.shape[0], 1, dtype=torch.float32)
|
||||
scaled_mm = torch._scaled_mm(
|
||||
@@ -147,9 +139,7 @@ class TestScaledMMRSModel(_BaseScaledMMModel):
|
||||
|
||||
class TestAGScaledMMModel(_BaseScaledMMModel):
|
||||
def forward(self, input: torch.Tensor):
|
||||
"""
|
||||
Forward pass implementing the all gather + scaled_mm in the FX graph
|
||||
"""
|
||||
"""Forward pass implementing the all gather + scaled_mm in the FX graph."""
|
||||
# Reshape input
|
||||
fp8_input = input.to(FP8_DTYPE)
|
||||
all_gather = tensor_model_parallel_all_gather(fp8_input, dim=0)
|
||||
@@ -173,8 +163,7 @@ class TestAGScaledMMModel(_BaseScaledMMModel):
|
||||
|
||||
class TestCutlassScaledMMRSModel(_BaseScaledMMModel):
|
||||
def forward(self, input: torch.Tensor):
|
||||
"""
|
||||
Forward pass implementing the cutlass_scaled_mm + reduce scatter
|
||||
"""Forward pass implementing the cutlass_scaled_mm + reduce scatter
|
||||
in the FX graph
|
||||
|
||||
"""
|
||||
@@ -200,8 +189,7 @@ class TestCutlassScaledMMRSModel(_BaseScaledMMModel):
|
||||
|
||||
class TestAGCutlassScaledMMModel(_BaseScaledMMModel):
|
||||
def forward(self, input: torch.Tensor):
|
||||
"""
|
||||
Forward pass implementing the all gather + cutlass_scaled_mm
|
||||
"""Forward pass implementing the all gather + cutlass_scaled_mm
|
||||
in the FX graph
|
||||
"""
|
||||
# Reshape input
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Comprehensive tests for UnsafeCloneEliminationPass.
|
||||
"""Comprehensive tests for UnsafeCloneEliminationPass.
|
||||
|
||||
This test suite exercises all possible valid FX graph patterns involving clones:
|
||||
1. Clone with no users (dead code)
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Tests for IR inplace functionalization pass integration.
|
||||
"""Tests for IR inplace functionalization pass integration.
|
||||
|
||||
This test suite verifies that the inplace functionalization pass, lowering pass,
|
||||
and clone cleanup pass work together correctly with donated buffer tracking.
|
||||
@@ -108,8 +107,7 @@ class MixedModel(nn.Module):
|
||||
|
||||
|
||||
class ModelWithTritonAfterMaybeInplace(nn.Module):
|
||||
"""
|
||||
Model using maybe_inplace followed by a Triton kernel.
|
||||
"""Model using maybe_inplace followed by a Triton kernel.
|
||||
Test clone elimination can handle Triton in the graph
|
||||
"""
|
||||
|
||||
@@ -410,8 +408,7 @@ def with_dyn_arg(fn: Callable, arg_index: int, dim_index: int):
|
||||
reason="Only test on cuda and rocm platform",
|
||||
)
|
||||
def test_piecewise_compilation_with_donated_buffers(monkeypatch, fresh_vllm_cache):
|
||||
"""
|
||||
Test piecewise compilation with donated buffers across graph splits.
|
||||
"""Test piecewise compilation with donated buffers across graph splits.
|
||||
Utilizes a custom splitting op. Uses fresh cache to avoid compilation caching.
|
||||
"""
|
||||
torch.set_default_device(current_platform.device_type)
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Unit tests for the DoubleQuant fan-out variants registered by
|
||||
"""Unit tests for the DoubleQuant fan-out variants registered by
|
||||
``RocmAiterRMSNormQuantFusionPass``.
|
||||
|
||||
Both variants target a 1-to-2 fan-out where one ``rms_norm`` output feeds
|
||||
@@ -90,8 +89,7 @@ def test_double_aiter_rms_fp8_group_quant_fusion(
|
||||
model_cls: type[torch.nn.Module],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""
|
||||
Both fan-out shapes (with and without an intermediate view) must fuse
|
||||
"""Both fan-out shapes (with and without an intermediate view) must fuse
|
||||
into ``rocm_aiter_rmsnorm_fp8_group_quant``: the no-view shape via
|
||||
``DoubleAiterRMSFp8GroupQuantPattern`` and the viewed shape via the
|
||||
new ``DoubleAiterRMSFp8GroupQuantViewPattern`` sibling.
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Unit test for the MLADualRMSNormFusionPass.
|
||||
"""Unit test for the MLADualRMSNormFusionPass.
|
||||
|
||||
The pass fuses paired q/kv RMS norms in MLA attention into a single
|
||||
fused_mla_dual_rms_norm op backed by AITER's fused_qk_rmsnorm kernel.
|
||||
@@ -38,13 +37,12 @@ FP8_DTYPE = current_platform.fp8_dtype()
|
||||
|
||||
|
||||
class MLADualRMSNormTestModel(torch.nn.Module):
|
||||
"""
|
||||
Minimal model reproducing the MLA dual RMS norm pattern:
|
||||
linear -> split([q_dim, kv_dim])
|
||||
+-- q_c (getitem 0) -> rms_norm(q_w, eps) -> linear
|
||||
+-- kv_lora (getitem 1) -> split([kv_c_dim, k_pe_dim])
|
||||
+-- kv_c (getitem 0) -> rms_norm(kv_w, eps)
|
||||
+-- k_pe
|
||||
"""Minimal model reproducing the MLA dual RMS norm pattern:
|
||||
linear -> split([q_dim, kv_dim])
|
||||
+-- q_c (getitem 0) -> rms_norm(q_w, eps) -> linear
|
||||
+-- kv_lora (getitem 1) -> split([kv_c_dim, k_pe_dim])
|
||||
+-- kv_c (getitem 0) -> rms_norm(kv_w, eps)
|
||||
+-- k_pe
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -155,13 +153,12 @@ def test_fuse_mla_dual_rms_norm(
|
||||
|
||||
|
||||
class MLADualRMSNormFp8PerTokenTestModel(torch.nn.Module):
|
||||
"""
|
||||
Minimal model reproducing the FP8 MLA attention path with *per-token* quant:
|
||||
linear -> split([q_dim, kv_dim])
|
||||
+-- q_c (getitem 0) -> rocm_aiter_rmsnorm_fused_dynamic_quant -> dequant
|
||||
+-- kv_lora (getitem 1) -> split([kv_c_dim, k_pe_dim])
|
||||
+-- kv_c (getitem 0) -> rms_norm (bf16)
|
||||
+-- k_pe
|
||||
"""Minimal model reproducing the FP8 MLA attention path with *per-token* quant:
|
||||
linear -> split([q_dim, kv_dim])
|
||||
+-- q_c (getitem 0) -> rocm_aiter_rmsnorm_fused_dynamic_quant -> dequant
|
||||
+-- kv_lora (getitem 1) -> split([kv_c_dim, k_pe_dim])
|
||||
+-- kv_c (getitem 0) -> rms_norm (bf16)
|
||||
+-- k_pe
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
|
||||
@@ -732,7 +732,8 @@ def test_aiter_fusion_rmsnorm_gated_quant_no_gdn_layers(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""Verify that without GDN layers in static_forward_context,
|
||||
the gated pattern is not registered and no matches occur."""
|
||||
the gated pattern is not registered and no matches occur.
|
||||
"""
|
||||
group_shape = GroupShape(1, 128)
|
||||
vllm_config = VllmConfig(
|
||||
model_config=ModelConfig(dtype=dtype),
|
||||
|
||||
@@ -107,7 +107,6 @@ class AttentionQuantPatternModel(torch.nn.Module):
|
||||
|
||||
def build_attn_metadata(self, batch_size: int) -> AttentionMetadata:
|
||||
"""Initialize attention metadata."""
|
||||
|
||||
# TODO (Rohan138) reuse utils from vllm/v1/worker/gpu/attn_utils.py
|
||||
|
||||
# Create common attn metadata
|
||||
@@ -304,7 +303,7 @@ def test_attention_quant_pattern(
|
||||
dist_init,
|
||||
disable_vllm_compile_cache,
|
||||
):
|
||||
"""Test AttentionStaticQuantPattern fusion pass"""
|
||||
"""Test AttentionStaticQuantPattern fusion pass."""
|
||||
if backend == AttentionBackendEnum.FLASHINFER and (
|
||||
not current_platform.is_device_capability((10, 0)) or not has_flashinfer()
|
||||
):
|
||||
|
||||
@@ -141,7 +141,6 @@ class MLAAttentionQuantPatternModel(torch.nn.Module):
|
||||
the full output[:num_actual_toks] buffer after both forward_mha and
|
||||
forward_mqa have written their results.
|
||||
"""
|
||||
|
||||
batch_spec = BatchSpec(seq_lens=[1] * batch_size, query_lens=[1] * batch_size)
|
||||
common_attn_metadata = create_common_attn_metadata(
|
||||
batch_spec, self.block_size, self.device, arange_block_indices=True
|
||||
@@ -408,7 +407,7 @@ def test_mla_attention_quant_pattern(
|
||||
dist_init,
|
||||
disable_vllm_compile_cache,
|
||||
):
|
||||
"""Test MLA AttentionQuantPattern fusion pass"""
|
||||
"""Test MLA AttentionQuantPattern fusion pass."""
|
||||
if (
|
||||
model_class is TestMLAAttentionNvfp4QuantPatternModel
|
||||
and not is_nvfp4_supported()
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Tests for manual fusion via maybe_fused_act_quant.
|
||||
"""Tests for manual fusion via maybe_fused_act_quant.
|
||||
|
||||
Tests all fusion paths in _FUSED_ACT_QUANT:
|
||||
- kFp8StaticTensorSym: all platforms
|
||||
|
||||
@@ -15,7 +15,8 @@ DEVICE_TYPE = current_platform.device_type
|
||||
|
||||
class SplitCoalescingModel(torch.nn.Module):
|
||||
"""Model with 3 separate split_with_sizes calls on the same input,
|
||||
simulating the B200+FP8 graph where CSE fails to merge them."""
|
||||
simulating the B200+FP8 graph where CSE fails to merge them.
|
||||
"""
|
||||
|
||||
def __init__(self, q_size: int, kv_size: int) -> None:
|
||||
super().__init__()
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Shared PyTorch custom silly attention for compilation tests.
|
||||
"""Shared PyTorch custom silly attention for compilation tests.
|
||||
Centralizes custom operation definitions to avoid duplicate registrations.
|
||||
"""
|
||||
|
||||
@@ -21,12 +20,12 @@ _global_counter = 0
|
||||
|
||||
|
||||
def get_global_counter():
|
||||
"""Get the current global counter value"""
|
||||
"""Get the current global counter value."""
|
||||
return _global_counter
|
||||
|
||||
|
||||
def reset_global_counter():
|
||||
"""Reset the global counter to 0"""
|
||||
"""Reset the global counter to 0."""
|
||||
global _global_counter
|
||||
_global_counter = 0
|
||||
|
||||
@@ -34,8 +33,7 @@ def reset_global_counter():
|
||||
def silly_attention(
|
||||
q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, out: torch.Tensor
|
||||
) -> None:
|
||||
"""
|
||||
Unified attention implementation that depends on
|
||||
"""Unified attention implementation that depends on
|
||||
all inputs and affects the output.
|
||||
Always increments a global counter that tests can use or ignore.
|
||||
"""
|
||||
@@ -51,7 +49,7 @@ def silly_attention(
|
||||
def silly_attention_fake(
|
||||
q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, out: torch.Tensor
|
||||
) -> None:
|
||||
"""Fake implementation for testing"""
|
||||
"""Fake implementation for testing."""
|
||||
return
|
||||
|
||||
|
||||
|
||||
@@ -268,8 +268,7 @@ def test_save_and_load_slice(monkeypatch: pytest.MonkeyPatch):
|
||||
def test_cache_load_returns_tuple_consistency_tuple_output(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""
|
||||
Test that cache loading correctly handles models that return tuples.
|
||||
"""Test that cache loading correctly handles models that return tuples.
|
||||
|
||||
This verifies that when a model returns a tuple of tensors, the output
|
||||
type is preserved as a tuple between fresh compilation and cache load.
|
||||
@@ -335,8 +334,7 @@ def test_cache_load_returns_tuple_consistency_tuple_output(
|
||||
|
||||
@pytest.mark.skipif(not is_torch_equal_or_newer("2.10.0"), reason="requires torch 2.10")
|
||||
def test_shape_env(monkeypatch: pytest.MonkeyPatch):
|
||||
"""
|
||||
Test that the shape environment is correctly serialized and preserved
|
||||
"""Test that the shape environment is correctly serialized and preserved
|
||||
when loading from cache.
|
||||
"""
|
||||
with monkeypatch.context() as m:
|
||||
@@ -374,8 +372,7 @@ def test_shape_env(monkeypatch: pytest.MonkeyPatch):
|
||||
def test_partition_wrapper_applied_on_aot_load(
|
||||
monkeypatch: pytest.MonkeyPatch, vllm_tmp_cache: Path, mocker
|
||||
):
|
||||
"""
|
||||
Test that partition wrappers are applied when loading AOT cached functions.
|
||||
"""Test that partition wrappers are applied when loading AOT cached functions.
|
||||
|
||||
This test verifies the fix for GitHub issue #31439 where AOT compile
|
||||
caused 2x latency regression when use_inductor_graph_partition=True.
|
||||
@@ -498,14 +495,12 @@ def test_standalone_compile_correctness():
|
||||
@pytest.mark.skipif(not is_torch_equal_or_newer("2.10.0"), reason="requires torch 2.10")
|
||||
@create_new_process_for_each_test("spawn")
|
||||
def test_gpt2_cache_hit(monkeypatch: pytest.MonkeyPatch):
|
||||
"""
|
||||
Test that compiling gpt2 twice results in a cache hit.
|
||||
"""Test that compiling gpt2 twice results in a cache hit.
|
||||
|
||||
Counter values are read from the EngineCore subprocess via
|
||||
``LLM.collective_rpc`` so the test works under default V1
|
||||
multiprocessing (no shared memory between test and engine).
|
||||
"""
|
||||
|
||||
from vllm import LLM
|
||||
|
||||
def _snap(self):
|
||||
@@ -641,7 +636,7 @@ class TestStandaloneCompiledArtifacts:
|
||||
|
||||
@patch("torch._inductor.standalone_compile.AOTCompiledArtifact.deserialize")
|
||||
def test_load_all_success(self, mock_deserialize):
|
||||
"""Test successful loading of all artifacts"""
|
||||
"""Test successful loading of all artifacts."""
|
||||
cache = StandaloneCompiledArtifacts()
|
||||
|
||||
mock_artifact1 = Mock()
|
||||
@@ -658,7 +653,7 @@ class TestStandaloneCompiledArtifacts:
|
||||
|
||||
@patch("torch._inductor.standalone_compile.AOTCompiledArtifact.deserialize")
|
||||
def test_load_all_already_loaded(self, mock_deserialize):
|
||||
"""Test that load_all skips if already loaded"""
|
||||
"""Test that load_all skips if already loaded."""
|
||||
cache = StandaloneCompiledArtifacts()
|
||||
|
||||
mock_artifact = Mock()
|
||||
@@ -671,7 +666,7 @@ class TestStandaloneCompiledArtifacts:
|
||||
|
||||
@patch("torch._inductor.standalone_compile.AOTCompiledArtifact.deserialize")
|
||||
def test_get_loaded_artifact(self, mock_deserialize):
|
||||
"""Test retrieving loaded artifacts"""
|
||||
"""Test retrieving loaded artifacts."""
|
||||
cache = StandaloneCompiledArtifacts()
|
||||
|
||||
mock_artifact = Mock()
|
||||
|
||||
@@ -77,7 +77,8 @@ def test_non_primitive_kwargs_lifted_to_consts(
|
||||
into the namespace, so ``device`` is unbound. The fix collects such
|
||||
objects into ``__vllm_consts__`` and references them by index. The
|
||||
unqualified ``device(type=...)`` form must never appear in the
|
||||
generated source."""
|
||||
generated source.
|
||||
"""
|
||||
split_gm = _trace_and_split(model_fn, (x,), split_ops)
|
||||
code, submod_names, consts = generate_execution_code(split_gm)
|
||||
|
||||
@@ -98,7 +99,8 @@ def test_dtype_singleton_deduped(x: torch.Tensor) -> None:
|
||||
"""``torch.float16`` is a process-wide singleton, so two ops referring
|
||||
to it in the traced graph share a single consts slot via ``id()``-based
|
||||
dedup. Distinct expressions (``x.to(...)`` vs ``(x*2).to(...)``) ensure
|
||||
the tracer can't CSE the two ops into a single node."""
|
||||
the tracer can't CSE the two ops into a single node.
|
||||
"""
|
||||
|
||||
def model_fn(x: torch.Tensor) -> torch.Tensor:
|
||||
return x.to(dtype=torch.float16) + (x * 2).to(dtype=torch.float16)
|
||||
@@ -147,7 +149,8 @@ def test_distinct_dtypes_get_distinct_slots(x: torch.Tensor) -> None:
|
||||
def test_consts_ordering_deterministic(x: torch.Tensor) -> None:
|
||||
"""Two independent traces of the same model must produce equal consts
|
||||
lists *in the same order*. Cache artifacts identify const slots by
|
||||
index, so a non-deterministic order would invalidate cached code."""
|
||||
index, so a non-deterministic order would invalidate cached code.
|
||||
"""
|
||||
|
||||
def model_fn(x: torch.Tensor) -> torch.Tensor:
|
||||
# Multiple distinct non-primitives encountered in a fixed graph order.
|
||||
@@ -185,7 +188,8 @@ def test_consts_shared_across_split_submods(x: torch.Tensor) -> None:
|
||||
The function below splits into three inlined submods, two of which
|
||||
independently reference ``torch.float16``. The shared ``const_index``
|
||||
threaded through recursive ``generate_execution_code_with_name`` calls
|
||||
must collapse the dtype to a single slot used from both submods."""
|
||||
must collapse the dtype to a single slot used from both submods.
|
||||
"""
|
||||
|
||||
def model_fn(x: torch.Tensor) -> torch.Tensor:
|
||||
a = x.to(dtype=torch.float16) # submod_0: _to_copy(fp16)
|
||||
@@ -230,7 +234,8 @@ def test_non_graphmodule_submod_uses_indexed_callable(x: torch.Tensor) -> None:
|
||||
"""When a child of split_gm is *not* a ``torch.fx.GraphModule`` — as
|
||||
happens in production once ``PiecewiseBackend`` replaces submods —
|
||||
codegen emits ``__vllm_submods__[idx](...)`` instead of inlining, and
|
||||
the runtime callable is bound from ``submod_callables``."""
|
||||
the runtime callable is bound from ``submod_callables``.
|
||||
"""
|
||||
|
||||
def model_fn(x: torch.Tensor) -> torch.Tensor:
|
||||
return x.relu().sigmoid()
|
||||
@@ -280,7 +285,8 @@ def test_non_graphmodule_submod_uses_indexed_callable(x: torch.Tensor) -> None:
|
||||
def test_getitem_in_stitching_graph(x: torch.Tensor) -> None:
|
||||
"""``operator.getitem`` on submod tuple returns is the ``call_function``
|
||||
special case at codegen.py — emitted as ``name = source[index]``
|
||||
rather than a function call."""
|
||||
rather than a function call.
|
||||
"""
|
||||
|
||||
def model_fn(x: torch.Tensor) -> torch.Tensor:
|
||||
return x.relu().sigmoid()
|
||||
@@ -298,7 +304,8 @@ def test_getitem_in_stitching_graph(x: torch.Tensor) -> None:
|
||||
def test_del_emitted_for_intermediate_values(x: torch.Tensor) -> None:
|
||||
"""The codegen schedules ``del`` after a value's last use to free
|
||||
memory early. Multi-submod splits naturally have intermediates whose
|
||||
last use is not the output node."""
|
||||
last use is not the output node.
|
||||
"""
|
||||
|
||||
def model_fn(x: torch.Tensor) -> torch.Tensor:
|
||||
return x.relu().sigmoid().tanh()
|
||||
@@ -318,7 +325,8 @@ def test_with_submod_false_rejects_call_module() -> None:
|
||||
"""``generate_execution_code_with_name(with_submod=False)`` is the
|
||||
recursive entry for inlining a GraphModule into its parent. It must
|
||||
refuse a graph that itself contains ``call_module`` nodes — the parent
|
||||
is responsible for handling those."""
|
||||
is responsible for handling those.
|
||||
"""
|
||||
g = fx.Graph()
|
||||
x_node = g.placeholder("x")
|
||||
root = torch.nn.Module()
|
||||
@@ -335,7 +343,8 @@ def test_node_ref_recurses_through_containers() -> None:
|
||||
"""``_node_ref`` is the recursive walker that lifts non-primitives
|
||||
nested inside list/tuple/dict args. Real aten ops rarely produce such
|
||||
structures, but the path is needed for DTensor placement lists and
|
||||
other future cases — unit-test the walker directly."""
|
||||
other future cases — unit-test the walker directly.
|
||||
"""
|
||||
consts: list = []
|
||||
const_index: dict[int, int] = {}
|
||||
cpu = torch.device("cpu")
|
||||
@@ -359,7 +368,8 @@ def test_node_ref_recurses_through_containers() -> None:
|
||||
def test_legacy_code_without_consts() -> None:
|
||||
"""``compile_execution_fn(consts=None)`` must still load code that has
|
||||
no ``__vllm_consts__`` reference, so older serialized cache artifacts
|
||||
keep working."""
|
||||
keep working.
|
||||
"""
|
||||
# Pre-consts codegen: no __vllm_consts__ reference, only torch/operator.
|
||||
legacy_code = (
|
||||
"import torch\n"
|
||||
|
||||
@@ -129,7 +129,8 @@ def test_compile_config_get_compile_ranges():
|
||||
|
||||
class PostGradStaticShapeChecker(InductorPass):
|
||||
"""Asserts that compile_sizes entries produce graphs with fully concrete
|
||||
(non-symbolic) shapes, and compile_ranges entries have symbolic shapes."""
|
||||
(non-symbolic) shapes, and compile_ranges entries have symbolic shapes.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.num_static_calls = 0
|
||||
@@ -170,7 +171,8 @@ class PostGradStaticShapeChecker(InductorPass):
|
||||
|
||||
def test_compile_sizes_produce_static_shapes(disable_vllm_compile_cache):
|
||||
"""Verify that compile_sizes entries are compiled with fully concrete
|
||||
shapes (no SymInts), while compile_ranges entries retain dynamic shapes."""
|
||||
shapes (no SymInts), while compile_ranges entries retain dynamic shapes.
|
||||
"""
|
||||
checker = PostGradStaticShapeChecker()
|
||||
torch.set_default_device("cuda")
|
||||
vllm_config = VllmConfig(
|
||||
|
||||
@@ -1229,7 +1229,8 @@ def test_compile_sizes_padding_validation():
|
||||
|
||||
def test_inductor_asserts_default_disabled(monkeypatch):
|
||||
"""Test that inductor runtime asserts are disabled by default
|
||||
(INFO logging level) on torch < 2.12."""
|
||||
(INFO logging level) on torch < 2.12.
|
||||
"""
|
||||
monkeypatch.setenv("VLLM_LOGGING_LEVEL", "INFO")
|
||||
|
||||
import importlib
|
||||
@@ -1247,7 +1248,8 @@ def test_inductor_asserts_default_disabled(monkeypatch):
|
||||
|
||||
def test_inductor_asserts_enabled_in_debug(monkeypatch):
|
||||
"""Test that VLLM_LOGGING_LEVEL=DEBUG enables inductor runtime asserts
|
||||
on torch < 2.12."""
|
||||
on torch < 2.12.
|
||||
"""
|
||||
monkeypatch.setenv("VLLM_LOGGING_LEVEL", "DEBUG")
|
||||
|
||||
import importlib
|
||||
@@ -1283,7 +1285,8 @@ def test_get_inductor_factors_includes_configs():
|
||||
|
||||
def test_inductor_asserts_user_override(monkeypatch):
|
||||
"""Test that explicit inductor_compile_config overrides the
|
||||
debug-logging default."""
|
||||
debug-logging default.
|
||||
"""
|
||||
monkeypatch.setenv("VLLM_LOGGING_LEVEL", "INFO")
|
||||
|
||||
import importlib
|
||||
|
||||
@@ -22,7 +22,7 @@ from vllm.utils.torch_utils import is_torch_equal_or_newer
|
||||
|
||||
|
||||
def get_test_models():
|
||||
"""Get list of models to test based on PyTorch version"""
|
||||
"""Get list of models to test based on PyTorch version."""
|
||||
models = [
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"openai-community/gpt2",
|
||||
@@ -172,7 +172,6 @@ def test_model_specialization_with_evaluate_guards(
|
||||
"""Test that evaluate_guards correctly detects shape specialization
|
||||
violations.
|
||||
"""
|
||||
|
||||
if (
|
||||
use_aot_compile
|
||||
and dynamic_shapes_type == DynamicShapesType.BACKED
|
||||
|
||||
@@ -24,8 +24,7 @@ DEVICE_TYPE = current_platform.device_type
|
||||
|
||||
|
||||
def test_getitem_moved_to_producer_subgraph():
|
||||
"""
|
||||
Test that getitem operations are moved to the same subgraph as their input,
|
||||
"""Test that getitem operations are moved to the same subgraph as their input,
|
||||
preventing tuple inputs to submodules.
|
||||
"""
|
||||
|
||||
@@ -79,8 +78,7 @@ def test_getitem_moved_to_producer_subgraph():
|
||||
|
||||
|
||||
def test_no_tuple_inputs_with_multiple_consumers():
|
||||
"""
|
||||
Test that when a tuple is consumed by multiple split operations,
|
||||
"""Test that when a tuple is consumed by multiple split operations,
|
||||
getitem operations are properly moved to avoid tuple inputs.
|
||||
"""
|
||||
|
||||
@@ -138,13 +136,10 @@ def test_no_tuple_inputs_with_multiple_consumers():
|
||||
|
||||
|
||||
def test_consecutive_ops_in_split():
|
||||
"""
|
||||
Test that consecutive splitting operations are grouped into the same subgraph
|
||||
"""
|
||||
"""Test that consecutive splitting operations are grouped into the same subgraph."""
|
||||
|
||||
def model_fn(x: torch.Tensor) -> torch.Tensor:
|
||||
"""
|
||||
Define a simple model where consecutive operations create opportunities
|
||||
"""Define a simple model where consecutive operations create opportunities
|
||||
for splitting subgraphs.
|
||||
"""
|
||||
# Apply silly attention followed by consecutive operations
|
||||
@@ -211,8 +206,7 @@ def _subgraphs_with_empty_nodes(split_items, *, is_splitting_graph):
|
||||
|
||||
|
||||
def test_empty_only_partition_stays_separate_after_splitting_predecessor():
|
||||
"""
|
||||
Empty-only subgraphs should not be merged when the only predecessor is
|
||||
"""Empty-only subgraphs should not be merged when the only predecessor is
|
||||
a splitting-op subgraph.
|
||||
"""
|
||||
|
||||
@@ -248,8 +242,7 @@ def test_empty_only_partition_stays_separate_after_splitting_predecessor():
|
||||
|
||||
|
||||
def test_empty_only_partition_is_merged():
|
||||
"""
|
||||
Empty-only subgraphs should still be merged when a non-splitting predecessor
|
||||
"""Empty-only subgraphs should still be merged when a non-splitting predecessor
|
||||
exists. The merged empty node must remain outside splitting-op subgraphs.
|
||||
"""
|
||||
|
||||
@@ -294,8 +287,7 @@ def test_empty_only_partition_is_merged():
|
||||
|
||||
|
||||
def test_builtin_empty_only_partition_is_merged():
|
||||
"""
|
||||
In Dynamo graphs, torch.empty/empty_like may appear as builtin call targets
|
||||
"""In Dynamo graphs, torch.empty/empty_like may appear as builtin call targets
|
||||
(not aten OpOverload). Ensure empty-only partitions are still merged.
|
||||
"""
|
||||
|
||||
@@ -340,8 +332,7 @@ def test_builtin_empty_only_partition_is_merged():
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_sym_size_whole_shape_boundary():
|
||||
"""
|
||||
Test that using x.size() (whole shape) across a split boundary can be
|
||||
"""Test that using x.size() (whole shape) across a split boundary can be
|
||||
compiled by standalone_compile.
|
||||
|
||||
The dynamo graph looks like:
|
||||
@@ -400,8 +391,7 @@ def test_sym_size_whole_shape_boundary():
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_symint_crosses_split_boundary():
|
||||
"""
|
||||
Test that SymInt placeholders from torch.compile + mark_dynamic
|
||||
"""Test that SymInt placeholders from torch.compile + mark_dynamic
|
||||
cross split boundaries safely via split_module's natural threading.
|
||||
|
||||
SymInt values are threaded through subgraphs by split_module and
|
||||
@@ -459,8 +449,7 @@ def test_symint_crosses_split_boundary():
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_shape_boundary_standalone_compile():
|
||||
"""
|
||||
Repro for the original production bug:
|
||||
"""Repro for the original production bug:
|
||||
|
||||
AssertionError: out_spec mismatch
|
||||
TreeSpec(tuple, None, [*, *, TreeSpec(Size, None, [*, *]), *])
|
||||
@@ -529,8 +518,7 @@ def test_shape_boundary_standalone_compile():
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_size_used_in_multiple_consumer_subgraphs():
|
||||
"""
|
||||
Validates that x.size() (whole shape) used by multiple downstream subgraphs
|
||||
"""Validates that x.size() (whole shape) used by multiple downstream subgraphs
|
||||
does not cause torch.Size to cross split boundaries.
|
||||
|
||||
Model:
|
||||
@@ -584,8 +572,7 @@ def test_size_used_in_multiple_consumer_subgraphs():
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_sym_size_metadata_propagated():
|
||||
"""
|
||||
Validates that new sym_size.int nodes created by the pre-pass have
|
||||
"""Validates that new sym_size.int nodes created by the pre-pass have
|
||||
example_value metadata set. Without it, placeholder metadata in consumer
|
||||
subgraphs would be None, breaking any code that dynamically builds
|
||||
example inputs from metadata (e.g. standalone_compile per-submodule).
|
||||
@@ -634,8 +621,7 @@ def test_sym_size_metadata_propagated():
|
||||
|
||||
|
||||
def test_decompose_size_with_getitem_user():
|
||||
"""
|
||||
Regression test: _decompose_size_nodes must handle getitem users of size()
|
||||
"""Regression test: _decompose_size_nodes must handle getitem users of size()
|
||||
correctly.
|
||||
|
||||
When a graph contains x.shape[i], it can appear as:
|
||||
@@ -704,8 +690,7 @@ def test_decompose_size_with_getitem_user():
|
||||
|
||||
|
||||
def test_decompose_size_leaves_scalar_size_with_dim():
|
||||
"""
|
||||
Regression test: _decompose_size_nodes must leave x.size(dim) alone.
|
||||
"""Regression test: _decompose_size_nodes must leave x.size(dim) alone.
|
||||
|
||||
x.size() returns a torch.Size tuple that can't cross split boundaries and
|
||||
must be decomposed. x.size(dim), however, already returns a scalar
|
||||
|
||||
@@ -236,7 +236,8 @@ class TestInductorFallbackFastPath:
|
||||
|
||||
def test_inner_set_membership_still_works_for_standard_ops(self):
|
||||
"""The patch must not break Inductor's existing fallback decisions
|
||||
for non-vllm ops such as ``torchvision::roi_align``."""
|
||||
for non-vllm ops such as ``torchvision::roi_align``.
|
||||
"""
|
||||
import torch._inductor.lowering as _lowering
|
||||
|
||||
allow_list = _lowering.FALLBACK_ALLOW_LIST
|
||||
|
||||
@@ -60,6 +60,7 @@ class TraceStructuredCapture:
|
||||
Args:
|
||||
event_type: The event type to filter by (e.g., "artifact", "graph_dump")
|
||||
name_pattern: Regex pattern to match against the artifact name
|
||||
|
||||
"""
|
||||
regex = re.compile(name_pattern)
|
||||
return [
|
||||
|
||||
@@ -40,7 +40,7 @@ class DummyLogprobsMode(Enum):
|
||||
|
||||
|
||||
def test_hash_factors_deterministic():
|
||||
"""Test that hash_factors produces consistent SHA-256 hashes"""
|
||||
"""Test that hash_factors produces consistent SHA-256 hashes."""
|
||||
factors = {"a": 1, "b": "test"}
|
||||
hash1 = hash_factors(factors)
|
||||
hash2 = hash_factors(factors)
|
||||
|
||||
@@ -372,7 +372,8 @@ class _HeterogeneousConfig(PretrainedConfig):
|
||||
"""A heterogeneous config with no convertor of its own.
|
||||
|
||||
Mirrors the parts vLLM uses: per-layer configs are shallow copies with the
|
||||
varying attributes applied and heterogeneity stripped, so they do not recurse."""
|
||||
varying attributes applied and heterogeneity stripped, so they do not recurse.
|
||||
"""
|
||||
|
||||
is_heterogeneous = True
|
||||
|
||||
@@ -471,7 +472,8 @@ def test_deepseek_v4_convertor_splits_vision_architecture():
|
||||
"""The vision checkpoint shares model_type/architectures with the
|
||||
text-only DeepSeek-V4; the convertor routes it to the VL wrapper class
|
||||
by rewriting hf_config.architectures (model-class resolution reads the
|
||||
raw attribute)."""
|
||||
raw attribute).
|
||||
"""
|
||||
from vllm.transformers_utils.configs.deepseek_v4 import DeepseekV4Config
|
||||
from vllm.transformers_utils.model_arch_config_convertor import (
|
||||
DeepseekV4ModelArchConfigConvertor,
|
||||
|
||||
@@ -9,12 +9,10 @@ from vllm.v1.engine.async_llm import AsyncLLM
|
||||
|
||||
|
||||
def test_mp_reducer():
|
||||
"""
|
||||
Test that _reduce_config reducer is registered when AsyncLLM is instantiated
|
||||
"""Test that _reduce_config reducer is registered when AsyncLLM is instantiated
|
||||
without transformers_modules. This is a regression test for
|
||||
https://github.com/vllm-project/vllm/pull/18640.
|
||||
"""
|
||||
|
||||
# Ensure transformers_modules is not in sys.modules
|
||||
if "transformers_modules" in sys.modules:
|
||||
del sys.modules["transformers_modules"]
|
||||
|
||||
@@ -42,7 +42,8 @@ def test_mm_encoder_attn_backend_hash_updates():
|
||||
|
||||
def test_language_model_only_does_not_affect_mm_hash():
|
||||
"""language_model_only does not affect the ViT computation graph,
|
||||
so it should not change the multimodal config hash."""
|
||||
so it should not change the multimodal config hash.
|
||||
"""
|
||||
base_hash = MultiModalConfig().compute_hash()
|
||||
lm_only_hash = MultiModalConfig(language_model_only=True).compute_hash()
|
||||
assert base_hash == lm_only_hash
|
||||
@@ -50,7 +51,8 @@ def test_language_model_only_does_not_affect_mm_hash():
|
||||
|
||||
def test_language_model_only_affects_model_hash():
|
||||
"""language_model_only affects the LM computation graph,
|
||||
so it should change the model config hash."""
|
||||
so it should change the model config hash.
|
||||
"""
|
||||
model = "llava-hf/llava-1.5-7b-hf"
|
||||
base_hash = ModelConfig(model).compute_hash()
|
||||
lm_only_hash = ModelConfig(model, language_model_only=True).compute_hash()
|
||||
@@ -386,7 +388,8 @@ def _resolve_mm_video_decode_device(
|
||||
torchcodec_available: bool = True,
|
||||
) -> dict:
|
||||
"""Run processor-device then video-decode-device resolution and report
|
||||
the resulting video media IO kwargs."""
|
||||
the resulting video media IO kwargs.
|
||||
"""
|
||||
mm_config = MultiModalConfig(
|
||||
mm_processor_kwargs={} if device is None else {"device": device},
|
||||
mm_tensor_ipc=mm_tensor_ipc, # type: ignore[arg-type]
|
||||
@@ -416,7 +419,8 @@ def _resolve_mm_video_decode_device(
|
||||
|
||||
def test_auto_video_decode_uses_nvdec_on_encoder_instance():
|
||||
"""The processor runs on the accelerator there, so decoded frames should
|
||||
stay on-device too."""
|
||||
stay on-device too.
|
||||
"""
|
||||
assert _resolve_mm_video_decode_device(ec_role="ec_producer") == {
|
||||
"backend": "torchcodec",
|
||||
"device": "cuda",
|
||||
@@ -428,7 +432,8 @@ def test_auto_video_decode_stays_on_cpu_off_encoder_instance(
|
||||
ec_role: ECRole | None,
|
||||
):
|
||||
"""The processor stays on CPU off encode-only instances, so video
|
||||
decoding should too."""
|
||||
decoding should too.
|
||||
"""
|
||||
assert _resolve_mm_video_decode_device(ec_role=ec_role) == {}
|
||||
|
||||
|
||||
|
||||
@@ -34,7 +34,8 @@ def _make_hf_config(**kwargs) -> PretrainedConfig:
|
||||
@pytest.mark.cpu_test
|
||||
def test_dict_overrides_are_not_forwarded_to_draft():
|
||||
"""Dict overrides are target-specific key patches; the draft must get
|
||||
only the architecture-mapping override."""
|
||||
only the architecture-mapping override.
|
||||
"""
|
||||
composed = SpeculativeConfig.compose_draft_hf_overrides(
|
||||
{"max_position_embeddings": 1234}
|
||||
)
|
||||
@@ -50,7 +51,8 @@ def test_none_overrides_fall_back_to_arch_mapping():
|
||||
@pytest.mark.cpu_test
|
||||
def test_callable_overrides_reach_the_draft_config():
|
||||
"""A callable override (config-to-config transform) composes with the
|
||||
architecture-mapping override and is applied to the draft config."""
|
||||
architecture-mapping override and is applied to the draft config.
|
||||
"""
|
||||
|
||||
def shrink(hf_config: PretrainedConfig) -> PretrainedConfig:
|
||||
hf_config.num_hidden_layers = 1
|
||||
@@ -67,7 +69,8 @@ def test_callable_overrides_reach_the_draft_config():
|
||||
@pytest.mark.cpu_test
|
||||
def test_arch_mapping_applies_before_callable_override():
|
||||
"""The static arch-mapping override runs first, so the user callable
|
||||
observes (and may adjust) the post-mapping config."""
|
||||
observes (and may adjust) the post-mapping config.
|
||||
"""
|
||||
seen_architectures: list[str] = []
|
||||
|
||||
def record(hf_config: PretrainedConfig) -> PretrainedConfig:
|
||||
@@ -128,7 +131,8 @@ def test_composed_override_is_picklable():
|
||||
the composed override must be picklable. A nested local closure is not
|
||||
(it raised ``Can't get local object`` on DFlashDraftModel); a
|
||||
``functools.partial`` over a module-referenceable static method is.
|
||||
Guard against regressing to a closure."""
|
||||
Guard against regressing to a closure.
|
||||
"""
|
||||
composed = SpeculativeConfig.compose_draft_hf_overrides(_module_level_shrink)
|
||||
|
||||
assert isinstance(composed, functools.partial)
|
||||
|
||||
@@ -99,7 +99,8 @@ def test_independent_draft_model_keeps_its_own_limit(
|
||||
vllm_caplog: pytest.LogCaptureFixture,
|
||||
):
|
||||
"""An independent AR draft may genuinely have a smaller context than the
|
||||
target; its max_position_embeddings must not be resized."""
|
||||
target; its max_position_embeddings must not be resized.
|
||||
"""
|
||||
target_model_config = ModelConfig(
|
||||
AR_MODEL, hf_overrides={"max_position_embeddings": 8192}
|
||||
)
|
||||
|
||||
@@ -11,7 +11,7 @@ from vllm.platforms import current_platform
|
||||
|
||||
|
||||
def check_cuda_context():
|
||||
"""Check CUDA driver context status"""
|
||||
"""Check CUDA driver context status."""
|
||||
try:
|
||||
cuda = ctypes.CDLL("libcuda.so")
|
||||
device = ctypes.c_int()
|
||||
@@ -22,7 +22,7 @@ def check_cuda_context():
|
||||
|
||||
|
||||
def run_cuda_test_in_thread(device_input, expected_device_id):
|
||||
"""Run CUDA context test in separate thread for isolation"""
|
||||
"""Run CUDA context test in separate thread for isolation."""
|
||||
try:
|
||||
# New thread should have no CUDA context initially
|
||||
valid_before, device_before = check_cuda_context()
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Test that platform imports do not prematurely initialize CUDA.
|
||||
"""Test that platform imports do not prematurely initialize CUDA.
|
||||
|
||||
This is critical for Ray-based multi-GPU setups where workers need to
|
||||
set CUDA_VISIBLE_DEVICES after importing vLLM but before CUDA is initialized.
|
||||
|
||||
@@ -44,8 +44,7 @@ def _make_request(stop, include_stop_str_in_output: bool, min_tokens: int = 0):
|
||||
|
||||
|
||||
def test_stop_string_while_stop_token_terminates(include_stop_str_in_output: bool):
|
||||
"""
|
||||
This test verifies that the detokenizer correctly handles the case where
|
||||
"""This test verifies that the detokenizer correctly handles the case where
|
||||
the generated token sequence contains both:
|
||||
- a stop token
|
||||
- an <eos> token
|
||||
@@ -67,7 +66,6 @@ def test_stop_string_while_stop_token_terminates(include_stop_str_in_output: boo
|
||||
where stop-string evaluation occurs before the early-return on
|
||||
stop_terminated.
|
||||
"""
|
||||
|
||||
# Generate text "abcdeZ" and tokenize it.
|
||||
generated_text = "abcde"
|
||||
eos_token = "Z"
|
||||
|
||||
@@ -18,13 +18,13 @@ DP_RANK = 0
|
||||
|
||||
@pytest.fixture
|
||||
def random_port():
|
||||
"""Generate a random port number for testing"""
|
||||
"""Generate a random port number for testing."""
|
||||
return random.randint(10000, 59900)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def publisher_config(random_port, request):
|
||||
"""Create a publisher config with inproc transport"""
|
||||
"""Create a publisher config with inproc transport."""
|
||||
how = request.param if hasattr(request, "param") else "inproc"
|
||||
|
||||
if how == "inproc":
|
||||
@@ -47,7 +47,7 @@ def publisher_config(random_port, request):
|
||||
|
||||
@pytest.fixture
|
||||
def publisher(publisher_config):
|
||||
"""Create and return a publisher instance"""
|
||||
"""Create and return a publisher instance."""
|
||||
pub = EventPublisherFactory.create(publisher_config, DP_RANK)
|
||||
yield pub
|
||||
pub.shutdown()
|
||||
@@ -55,7 +55,7 @@ def publisher(publisher_config):
|
||||
|
||||
@pytest.fixture
|
||||
def subscriber(publisher_config):
|
||||
"""Create and return a subscriber for testing"""
|
||||
"""Create and return a subscriber for testing."""
|
||||
endpoint = publisher_config.endpoint
|
||||
replay_endpoint = publisher_config.replay_endpoint
|
||||
|
||||
@@ -74,7 +74,7 @@ def subscriber(publisher_config):
|
||||
|
||||
|
||||
class MockSubscriber:
|
||||
"""Helper class to receive and verify published events"""
|
||||
"""Helper class to receive and verify published events."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -113,7 +113,7 @@ class MockSubscriber:
|
||||
self.decoder = msgspec.msgpack.Decoder(type=decode_type)
|
||||
|
||||
def receive_one(self, timeout=1000) -> tuple[int, SampleBatch] | None:
|
||||
"""Receive a single message with timeout"""
|
||||
"""Receive a single message with timeout."""
|
||||
if not self.sub.poll(timeout):
|
||||
return None
|
||||
|
||||
@@ -127,7 +127,7 @@ class MockSubscriber:
|
||||
return seq, data
|
||||
|
||||
def request_replay(self, start_seq: int, socket_idx: int = 0) -> None:
|
||||
"""Request replay of messages starting from start_seq"""
|
||||
"""Request replay of messages starting from start_seq."""
|
||||
if not self.replay_sockets:
|
||||
raise ValueError("Replay sockets not initialized")
|
||||
if socket_idx >= len(self.replay_sockets):
|
||||
@@ -138,7 +138,7 @@ class MockSubscriber:
|
||||
)
|
||||
|
||||
def receive_replay(self, socket_idx: int = 0) -> list[tuple[int, SampleBatch]]:
|
||||
"""Receive replayed messages from a specific replay socket"""
|
||||
"""Receive replayed messages from a specific replay socket."""
|
||||
if not self.replay_sockets:
|
||||
raise ValueError("Replay sockets not initialized")
|
||||
if socket_idx >= len(self.replay_sockets):
|
||||
@@ -170,7 +170,7 @@ class MockSubscriber:
|
||||
return replayed
|
||||
|
||||
def close(self):
|
||||
"""Clean up resources"""
|
||||
"""Clean up resources."""
|
||||
self.sub.close()
|
||||
for replay in self.replay_sockets:
|
||||
replay.close()
|
||||
@@ -179,7 +179,8 @@ class MockSubscriber:
|
||||
@pytest.fixture
|
||||
def enable_ray_v2_backend():
|
||||
"""Set env vars for the Ray V2 executor backend and shut down Ray
|
||||
between tests."""
|
||||
between tests.
|
||||
"""
|
||||
import ray
|
||||
|
||||
saved = {
|
||||
|
||||
@@ -1,11 +1,10 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
WARNING: This test runs in both single-node (4 GPUs) and multi-node
|
||||
(2 node with 2 GPUs each) modes. If the test only uses 2 GPUs, it is
|
||||
important to set the distributed backend to "mp" to avoid Ray scheduling
|
||||
all workers in a node other than the head node, which can cause the test
|
||||
to fail.
|
||||
"""WARNING: This test runs in both single-node (4 GPUs) and multi-node
|
||||
(2 node with 2 GPUs each) modes. If the test only uses 2 GPUs, it is
|
||||
important to set the distributed backend to "mp" to avoid Ray scheduling
|
||||
all workers in a node other than the head node, which can cause the test
|
||||
to fail.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
@@ -29,7 +29,7 @@ def test_eplb_load_stats_reduce_across_ranks():
|
||||
|
||||
|
||||
def test_basic_rebalance():
|
||||
"""Test basic rebalancing functionality"""
|
||||
"""Test basic rebalancing functionality."""
|
||||
# Example from https://github.com/deepseek-ai/eplb
|
||||
weight = torch.tensor(
|
||||
[
|
||||
@@ -92,7 +92,7 @@ def test_basic_rebalance():
|
||||
|
||||
|
||||
def test_single_gpu_case():
|
||||
"""Test single GPU case"""
|
||||
"""Test single GPU case."""
|
||||
weight = torch.tensor([[10, 20, 30, 40]])
|
||||
num_replicas = 4
|
||||
num_groups = 1
|
||||
@@ -115,7 +115,7 @@ def test_single_gpu_case():
|
||||
|
||||
|
||||
def test_equal_weights():
|
||||
"""Test case with equal weights"""
|
||||
"""Test case with equal weights."""
|
||||
weight = torch.tensor([[50, 50, 50, 50, 50, 50, 50, 50]])
|
||||
num_replicas = 8
|
||||
num_groups = 2
|
||||
@@ -139,7 +139,7 @@ def test_equal_weights():
|
||||
|
||||
|
||||
def test_extreme_weight_imbalance():
|
||||
"""Test extreme weight imbalance case"""
|
||||
"""Test extreme weight imbalance case."""
|
||||
weight = torch.tensor([[1000, 1, 1, 1, 1, 1, 1, 1]])
|
||||
num_replicas = 12
|
||||
num_groups = 2
|
||||
@@ -162,7 +162,7 @@ def test_extreme_weight_imbalance():
|
||||
|
||||
|
||||
def test_multiple_layers():
|
||||
"""Test multiple layers case"""
|
||||
"""Test multiple layers case."""
|
||||
weight = torch.tensor(
|
||||
[
|
||||
[10, 20, 30, 40, 50, 60], # First layer
|
||||
@@ -195,7 +195,7 @@ def test_multiple_layers():
|
||||
|
||||
|
||||
def test_parameter_validation():
|
||||
"""Test parameter validation"""
|
||||
"""Test parameter validation."""
|
||||
weight = torch.tensor([[10, 20, 30, 40]])
|
||||
|
||||
# Test non-divisible case - this should handle normally without throwing
|
||||
@@ -213,7 +213,7 @@ def test_parameter_validation():
|
||||
|
||||
|
||||
def test_small_scale_hierarchical():
|
||||
"""Test small-scale hierarchical load balancing"""
|
||||
"""Test small-scale hierarchical load balancing."""
|
||||
weight = torch.tensor(
|
||||
[
|
||||
[100, 50, 200, 75, 150, 25, 300, 80], # 8 experts
|
||||
@@ -243,7 +243,7 @@ def test_small_scale_hierarchical():
|
||||
|
||||
|
||||
def test_global_load_balance_fallback():
|
||||
"""Test global load balancing fallback case"""
|
||||
"""Test global load balancing fallback case."""
|
||||
# When num_groups % num_nodes != 0, should fall back to global load
|
||||
# balancing
|
||||
weight = torch.tensor([[10, 20, 30, 40, 50, 60]])
|
||||
@@ -265,7 +265,7 @@ def test_global_load_balance_fallback():
|
||||
|
||||
@pytest.mark.parametrize("device", ["cpu", "cuda"])
|
||||
def test_device_compatibility(device):
|
||||
"""Test device compatibility"""
|
||||
"""Test device compatibility."""
|
||||
if device == "cuda" and not torch.cuda.is_available():
|
||||
pytest.skip("CUDA not available")
|
||||
|
||||
@@ -287,8 +287,7 @@ def test_device_compatibility(device):
|
||||
|
||||
|
||||
def test_additional_cases():
|
||||
"""Test more edge cases and different parameter combinations"""
|
||||
|
||||
"""Test more edge cases and different parameter combinations."""
|
||||
# Test case 1: Large-scale distributed setup
|
||||
weight1 = torch.tensor(
|
||||
[[50, 100, 75, 120, 90, 60, 80, 110, 40, 70, 95, 85, 65, 55, 45, 35]]
|
||||
@@ -320,8 +319,7 @@ def test_additional_cases():
|
||||
|
||||
|
||||
def test_compute_logical_maps_with_negative_indices():
|
||||
"""
|
||||
Test that compute_logical_maps correctly handles physical slots containing
|
||||
"""Test that compute_logical_maps correctly handles physical slots containing
|
||||
-1 (unused slots).
|
||||
"""
|
||||
# 2 layers, 6 physical slots, 4 logical experts.
|
||||
|
||||
@@ -57,9 +57,8 @@ def test_reuse_across_multiple_cycles():
|
||||
|
||||
|
||||
def test_producer_consumer():
|
||||
"""
|
||||
This test uses the CpuGpuEvent to synchronize reads and writes to/from a shared GPU
|
||||
tensor on multiple CPU threads.
|
||||
"""This test uses the CpuGpuEvent to synchronize reads and writes to/from a shared
|
||||
GPU tensor on multiple CPU threads.
|
||||
"""
|
||||
worker_stream = torch.cuda.Stream()
|
||||
# Create a single element counter that will be shared between two threads
|
||||
|
||||
@@ -56,8 +56,7 @@ def create_expert_indices_with_redundancy(
|
||||
total_physical_experts: int,
|
||||
redundancy_config: list[int], # redundancy for each logical expert
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Create expert indices with redundancy.
|
||||
"""Create expert indices with redundancy.
|
||||
|
||||
Args:
|
||||
num_layers: number of layers
|
||||
@@ -67,6 +66,7 @@ def create_expert_indices_with_redundancy(
|
||||
|
||||
Returns:
|
||||
indices: Shape (num_layers, total_physical_experts)
|
||||
|
||||
"""
|
||||
assert sum(redundancy_config) == total_physical_experts
|
||||
assert len(redundancy_config) == num_logical_experts
|
||||
@@ -95,8 +95,7 @@ def create_expert_weights(
|
||||
device: torch.device,
|
||||
physical_to_logical_mapping: torch.Tensor,
|
||||
) -> list[list[torch.Tensor]]:
|
||||
"""
|
||||
Create fake expert weights tensor for testing.
|
||||
"""Create fake expert weights tensor for testing.
|
||||
|
||||
Use `arange` to generate predictable weights values, based on logical
|
||||
expert ID.
|
||||
@@ -105,6 +104,7 @@ def create_expert_weights(
|
||||
Args:
|
||||
physical_to_logical_mapping: Shape (num_layers, num_local_experts)
|
||||
mapping[layer, physical_pos] = logical_expert_id
|
||||
|
||||
"""
|
||||
expert_weights = []
|
||||
|
||||
@@ -209,9 +209,7 @@ def verify_redundant_experts_have_same_weights(
|
||||
world_size: int,
|
||||
num_local_experts: int,
|
||||
) -> bool:
|
||||
"""
|
||||
Verify that all replicas of the same logical expert have the same weights.
|
||||
"""
|
||||
"""Verify that all replicas of the same logical expert have the same weights."""
|
||||
num_layers = len(expert_weights)
|
||||
total_physical_experts = world_size * num_local_experts
|
||||
|
||||
@@ -568,7 +566,6 @@ def test_rearrange_expert_weights_with_redundancy(
|
||||
eplb_communicator,
|
||||
):
|
||||
"""Test the functionality of rearranging expert weights with redundancy."""
|
||||
|
||||
if eplb_communicator == "nixl" and not has_nixl():
|
||||
pytest.skip("NIXL is not available")
|
||||
if torch.accelerator.device_count() < world_size:
|
||||
@@ -678,7 +675,6 @@ def test_async_transfer_layer_without_mtp(
|
||||
eplb_communicator: str,
|
||||
):
|
||||
"""Exercise async EPLB transfer path without MTP/spec decode."""
|
||||
|
||||
if eplb_communicator == "nixl" and not has_nixl():
|
||||
pytest.skip("NIXL is not available")
|
||||
if torch.accelerator.device_count() < world_size:
|
||||
@@ -696,11 +692,9 @@ def test_async_transfer_layer_without_mtp(
|
||||
|
||||
@pytest.mark.parametrize("world_size", [2, 4])
|
||||
def test_rearrange_expert_weights_no_change(world_size):
|
||||
"""
|
||||
Test that when the indices do not change, the weights should remain
|
||||
"""Test that when the indices do not change, the weights should remain
|
||||
unchanged.
|
||||
"""
|
||||
|
||||
if torch.accelerator.device_count() < world_size:
|
||||
pytest.skip(f"Need at least {world_size} GPUs to run the test")
|
||||
distributed_run(
|
||||
@@ -800,7 +794,6 @@ def _test_rearrange_expert_weights_profile_mode(env, world_size) -> None:
|
||||
@pytest.mark.parametrize("world_size", [2, 4])
|
||||
def test_rearrange_expert_weights_profile_mode(world_size):
|
||||
"""Test profile mode (should not copy actual weights)"""
|
||||
|
||||
if torch.accelerator.device_count() < world_size:
|
||||
pytest.skip(f"Need at least {world_size} GPUs to run the test")
|
||||
distributed_run(
|
||||
|
||||
@@ -124,7 +124,8 @@ def _build_processed_layer(
|
||||
backend: str, raw: dict[str, torch.Tensor], device: torch.device
|
||||
) -> tuple[CompressedTensorsW4A4Nvfp4MoEMethod, _RoutedExpertsStub]:
|
||||
"""Create the method + layer, load ``raw`` and run the real
|
||||
process_weights_after_loading (which builds the flashinfer kernel)."""
|
||||
process_weights_after_loading (which builds the flashinfer kernel).
|
||||
"""
|
||||
moe_config = _make_moe_config(backend)
|
||||
method = CompressedTensorsW4A4Nvfp4MoEMethod(moe_config, "layer.0", use_a16=False)
|
||||
layer = _RoutedExpertsStub(moe_config).to(device)
|
||||
@@ -157,7 +158,8 @@ def _assert_tensors_equal(
|
||||
|
||||
def _simulate_eplb_rearrangement(layer: torch.nn.Module, perm: torch.Tensor) -> None:
|
||||
"""Permute experts of every registered Parameter in place, the way
|
||||
EPLB's in-place rearrangement moves expert slices along dim 0."""
|
||||
EPLB's in-place rearrangement moves expert slices along dim 0.
|
||||
"""
|
||||
with torch.no_grad():
|
||||
for _, param in layer.named_parameters():
|
||||
param.copy_(param[perm])
|
||||
@@ -165,7 +167,8 @@ def _simulate_eplb_rearrangement(layer: torch.nn.Module, perm: torch.Tensor) ->
|
||||
|
||||
def _ensure_world1_distributed() -> None:
|
||||
"""The enable_eplb config makes weight processing all-reduce activation
|
||||
scale amaxes over the EP group, so a (single-rank) EP group must exist."""
|
||||
scale amaxes over the EP group, so a (single-rank) EP group must exist.
|
||||
"""
|
||||
from vllm.distributed.parallel_state import (
|
||||
ensure_model_parallel_initialized,
|
||||
init_distributed_environment,
|
||||
@@ -253,7 +256,8 @@ def test_nvfp4_eplb_rearrangement_matches_reload(backend: str) -> None:
|
||||
def test_fp8_per_tensor_alphas_registered_and_aliased() -> None:
|
||||
"""CPU-only: the fp8 per-tensor oracle must register the fused
|
||||
(w_scale * a_scale) products as layer Parameters aliased by the quant
|
||||
config, so EPLB rearrangement keeps kernels consistent."""
|
||||
config, so EPLB rearrangement keeps kernels consistent.
|
||||
"""
|
||||
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
|
||||
Fp8MoeBackend,
|
||||
make_fp8_moe_quant_config,
|
||||
|
||||
@@ -75,8 +75,7 @@ def test_eplb_spec_decode(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
model_setup: tuple[str, str, str, int, float],
|
||||
):
|
||||
"""
|
||||
Test the correctness of EPLB speculative decoding with GSM8K dataset.
|
||||
"""Test the correctness of EPLB speculative decoding with GSM8K dataset.
|
||||
Applicable to MoE models with mtp or eagle spec decode.
|
||||
"""
|
||||
method, model_name, spec_model_name, tp_size, expected_gsm8k_value = model_setup
|
||||
@@ -109,10 +108,7 @@ def test_eplb_spec_decode(
|
||||
|
||||
@large_gpu_mark(min_gb=80)
|
||||
def test_eplb_spec_decode_qwen3_next_mtp_async() -> None:
|
||||
"""
|
||||
Ensure async EPLB works with MTP speculative decoding for Qwen3-Next.
|
||||
"""
|
||||
|
||||
"""Ensure async EPLB works with MTP speculative decoding for Qwen3-Next."""
|
||||
TASK = "gsm8k"
|
||||
FILTER = "exact_match,strict-match"
|
||||
RTOL = 0.03
|
||||
|
||||
@@ -30,8 +30,7 @@ def _make_model_state(
|
||||
|
||||
|
||||
def test_commit_eplb_maps_shape_change():
|
||||
"""
|
||||
When the number of physical experts changes, resize the active map within its
|
||||
"""When the number of physical experts changes, resize the active map within its
|
||||
preallocated storage.
|
||||
"""
|
||||
num_layers, num_logical, num_physical = 2, 4, 6
|
||||
@@ -65,8 +64,7 @@ def test_commit_eplb_maps_shape_change():
|
||||
|
||||
|
||||
def test_commit_eplb_maps_for_layer_logical_padding():
|
||||
"""
|
||||
Test that logical_to_physical_map is padded with -1 to fill the
|
||||
"""Test that logical_to_physical_map is padded with -1 to fill the
|
||||
pre-allocated slots when the new map has fewer replicas than the max.
|
||||
"""
|
||||
num_layers, num_logical, num_physical = 2, 4, 6
|
||||
|
||||
@@ -20,27 +20,26 @@ class EventSample(
|
||||
tag=True, # type: ignore
|
||||
array_like=True, # type: ignore
|
||||
):
|
||||
"""Test event for publisher testing"""
|
||||
"""Test event for publisher testing."""
|
||||
|
||||
id: int
|
||||
value: str
|
||||
|
||||
|
||||
class SampleBatch(EventBatch):
|
||||
"""Test event batch for publisher testing"""
|
||||
"""Test event batch for publisher testing."""
|
||||
|
||||
events: list[EventSample]
|
||||
|
||||
|
||||
def create_test_events(count: int) -> SampleBatch:
|
||||
"""Create a batch of test events"""
|
||||
"""Create a batch of test events."""
|
||||
events = [EventSample(id=i, value=f"test-{i}") for i in range(count)]
|
||||
return SampleBatch(ts=time.time(), events=events)
|
||||
|
||||
|
||||
def test_basic_publishing(publisher, subscriber):
|
||||
"""Test basic event publishing works"""
|
||||
|
||||
"""Test basic event publishing works."""
|
||||
test_batch = create_test_events(5)
|
||||
publisher.publish(test_batch)
|
||||
|
||||
@@ -58,7 +57,7 @@ def test_basic_publishing(publisher, subscriber):
|
||||
|
||||
|
||||
def test_multiple_events(publisher, subscriber):
|
||||
"""Test publishing and receiving multiple event batches"""
|
||||
"""Test publishing and receiving multiple event batches."""
|
||||
for _ in range(10):
|
||||
batch = create_test_events(2)
|
||||
publisher.publish(batch)
|
||||
@@ -75,7 +74,7 @@ def test_multiple_events(publisher, subscriber):
|
||||
|
||||
|
||||
def test_replay_mechanism(publisher, subscriber):
|
||||
"""Test the replay mechanism works correctly"""
|
||||
"""Test the replay mechanism works correctly."""
|
||||
for _ in range(19):
|
||||
batch = create_test_events(1)
|
||||
publisher.publish(batch)
|
||||
@@ -96,7 +95,7 @@ def test_replay_mechanism(publisher, subscriber):
|
||||
|
||||
|
||||
def test_replay_includes_topic(publisher, subscriber, publisher_config):
|
||||
"""Test that replay responses include the topic, matching PUB format"""
|
||||
"""Test that replay responses include the topic, matching PUB format."""
|
||||
for _ in range(5):
|
||||
publisher.publish(create_test_events(1))
|
||||
|
||||
@@ -115,7 +114,7 @@ def test_replay_includes_topic(publisher, subscriber, publisher_config):
|
||||
|
||||
|
||||
def test_buffer_limit(publisher, subscriber, publisher_config):
|
||||
"""Test buffer limit behavior"""
|
||||
"""Test buffer limit behavior."""
|
||||
buffer_size = publisher_config.buffer_steps
|
||||
|
||||
# Publish more events than the buffer can hold
|
||||
@@ -139,9 +138,7 @@ def test_buffer_limit(publisher, subscriber, publisher_config):
|
||||
|
||||
|
||||
def test_topic_filtering(publisher_config):
|
||||
"""
|
||||
Test that a subscriber only receives messages matching its topic filter
|
||||
"""
|
||||
"""Test that a subscriber only receives messages matching its topic filter."""
|
||||
publisher_config.replay_endpoint = None
|
||||
|
||||
publisher_config.topic = "foo"
|
||||
@@ -174,7 +171,7 @@ def test_topic_filtering(publisher_config):
|
||||
|
||||
|
||||
def test_high_volume(publisher, subscriber):
|
||||
"""Test publishing and receiving a high volume of events"""
|
||||
"""Test publishing and receiving a high volume of events."""
|
||||
num_batches = 10_000
|
||||
events_per_batch = 100
|
||||
|
||||
@@ -210,7 +207,7 @@ def test_high_volume(publisher, subscriber):
|
||||
|
||||
|
||||
def test_null_publisher():
|
||||
"""Test that NullEventPublisher can be used without errors"""
|
||||
"""Test that NullEventPublisher can be used without errors."""
|
||||
publisher = NullEventPublisher(DP_RANK)
|
||||
|
||||
# This should not raise any errors
|
||||
@@ -220,8 +217,7 @@ def test_null_publisher():
|
||||
|
||||
|
||||
def test_data_parallel_rank_tagging(publisher_config):
|
||||
"""Test that events are properly tagged with their data parallel rank"""
|
||||
|
||||
"""Test that events are properly tagged with their data parallel rank."""
|
||||
publisher_config.topic = "foo"
|
||||
pub_0 = EventPublisherFactory.create(publisher_config, DP_RANK)
|
||||
pub_1 = EventPublisherFactory.create(publisher_config, DP_RANK + 1)
|
||||
@@ -285,7 +281,7 @@ def test_data_parallel_rank_tagging(publisher_config):
|
||||
|
||||
|
||||
def test_event_publisher_factory(random_port):
|
||||
"""Test event publisher factory creation behavior under different configurations"""
|
||||
"""Test event publisher factory creation behavior under different configurations."""
|
||||
from vllm.config.kv_events import KVEventsConfig
|
||||
from vllm.distributed.kv_events import ZmqEventPublisher
|
||||
|
||||
|
||||
@@ -49,7 +49,6 @@ def verify_round_robin_pattern(expert_map, ep_rank, ep_size, global_num_experts)
|
||||
@pytest.mark.parametrize("world_size", [2, 4])
|
||||
def test_expert_placement_various_sizes(expert_placement_strategy, world_size):
|
||||
"""Test round_robin expert placement with various expert counts."""
|
||||
|
||||
# Test with different global_num_experts values
|
||||
# Include both divisible and non-divisible cases
|
||||
if world_size == 2:
|
||||
@@ -116,7 +115,6 @@ def test_expert_placement_various_sizes(expert_placement_strategy, world_size):
|
||||
@pytest.mark.parametrize("world_size", [2, 4])
|
||||
def test_expert_placement_edge_cases(expert_placement_strategy, world_size):
|
||||
"""Test edge cases for round_robin expert placement."""
|
||||
|
||||
# Test case 1: ep_size = 1 (should return None for expert_map)
|
||||
local_num_experts, expert_map, _ = determine_expert_map(
|
||||
ep_size=1,
|
||||
@@ -139,7 +137,6 @@ def test_expert_placement_edge_cases(expert_placement_strategy, world_size):
|
||||
|
||||
def test_determine_expert_map_comprehensive():
|
||||
"""Test of determine_expert_map function with various configurations."""
|
||||
|
||||
# Test cases: (ep_size, ep_rank, global_num_experts,
|
||||
# expert_placement_strategy, expected_local, expected_map_pattern)
|
||||
test_cases = [
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Tests for MNNVL AllToAll operations.
|
||||
"""Tests for MNNVL AllToAll operations.
|
||||
|
||||
Requires: docker run ... --cap-add=SYS_PTRACE ...
|
||||
Run: pytest tests/distributed/test_mnnvl_alltoall.py -v
|
||||
@@ -87,6 +86,7 @@ def _run_worker(rank, world_size, port, worker_fn, dp_size, dp_port, err_queue):
|
||||
Otherwise use tp=world_size (default for EP-based tests).
|
||||
dp_port: Separate port for the DP master (only used when dp_size is set).
|
||||
err_queue: Queue for propagating tracebacks to the parent process.
|
||||
|
||||
"""
|
||||
try:
|
||||
os.environ.pop("CUDA_VISIBLE_DEVICES", None)
|
||||
@@ -115,6 +115,7 @@ def _init_dp_environment(world_size, rank, port, dp_size, dp_port):
|
||||
Args:
|
||||
port: Port for torch.distributed init.
|
||||
dp_port: Separate port for the DP master group init.
|
||||
|
||||
"""
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.config.parallel import ParallelConfig
|
||||
@@ -159,7 +160,8 @@ def _make_forward_context(rank, world_size, num_tokens_per_rank):
|
||||
class _AttnMeta:
|
||||
"""Minimal placeholder so set_forward_context's
|
||||
``attn_metadata is not None`` guard (forward_context.py:334)
|
||||
is satisfied. The real DPMetadata is built from num_tokens_across_dp."""
|
||||
is satisfied. The real DPMetadata is built from num_tokens_across_dp.
|
||||
"""
|
||||
|
||||
dp_metadata = None
|
||||
|
||||
@@ -458,7 +460,8 @@ def _one_sided_workspace_grow_worker(rank, world_size):
|
||||
def test_one_sided_manager_workspace_grow(world_size):
|
||||
"""A later initialize() with a larger per-token payload must grow the
|
||||
workspace and rebuild MoeAlltoAll; a later initialize() with a smaller
|
||||
payload must no-op."""
|
||||
payload must no-op.
|
||||
"""
|
||||
_spawn_workers(
|
||||
_one_sided_workspace_grow_worker,
|
||||
world_size,
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Test that MessageQueue uses the local node's IP for binding,
|
||||
"""Test that MessageQueue uses the local node's IP for binding,
|
||||
not a remote master_addr. This validates the fix for cross-node
|
||||
data-parallel where each DP group leader must bind to its own IP.
|
||||
|
||||
@@ -22,7 +21,8 @@ from vllm.utils.network_utils import get_ip
|
||||
|
||||
def test_mq_bind_with_local_ip():
|
||||
"""MessageQueue with remote readers should successfully bind
|
||||
when connect_ip is the local node's IP."""
|
||||
when connect_ip is the local node's IP.
|
||||
"""
|
||||
# n_reader=2, n_local_reader=1 means 1 remote reader,
|
||||
# which triggers the remote ZMQ socket bind.
|
||||
mq = MessageQueue(
|
||||
@@ -44,7 +44,8 @@ def test_mq_bind_with_local_ip():
|
||||
def test_mq_bind_with_non_local_ip_fails():
|
||||
"""MessageQueue should fail to bind when connect_ip is a
|
||||
non-local IP address (simulating the bug where master_addr
|
||||
from a different node was used)."""
|
||||
from a different node was used).
|
||||
"""
|
||||
# Use a non-local IP that we definitely can't bind to.
|
||||
# 198.51.100.1 is from TEST-NET-2 (RFC 5737), never locally assigned.
|
||||
non_local_ip = "198.51.100.1"
|
||||
@@ -58,7 +59,8 @@ def test_mq_bind_with_non_local_ip_fails():
|
||||
|
||||
def test_mq_bind_defaults_to_local_ip():
|
||||
"""When connect_ip is None, MessageQueue should auto-detect
|
||||
the local IP and bind successfully."""
|
||||
the local IP and bind successfully.
|
||||
"""
|
||||
mq = MessageQueue(
|
||||
n_reader=2,
|
||||
n_local_reader=1,
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
Integration tests for MultiprocExecutor at the executor level.
|
||||
"""Integration tests for MultiprocExecutor at the executor level.
|
||||
This test directly tests the executor without going through the LLM interface,
|
||||
focusing on executor initialization, RPC calls, and distributed execution.
|
||||
"""
|
||||
@@ -377,8 +376,7 @@ def test_multiproc_executor_properties():
|
||||
|
||||
@multi_gpu_test(num_gpus=4)
|
||||
def test_multiproc_executor_multi_node():
|
||||
"""
|
||||
Test MultiprocExecutor with multi-node configuration.
|
||||
"""Test MultiprocExecutor with multi-node configuration.
|
||||
This simulates 2 nodes with TP=4:
|
||||
- Node 0 (rank 0): Uses GPUs 0,1 (CUDA_VISIBLE_DEVICES=0,1) with TP=2
|
||||
- Node 1 (rank 1): Uses GPUs 2,3 (CUDA_VISIBLE_DEVICES=2,3) with TP=2
|
||||
|
||||
@@ -1,11 +1,10 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
WARNING: This test runs in both single-node (4 GPUs) and multi-node
|
||||
(2 node with 2 GPUs each) modes. If the test only uses 2 GPUs, it is
|
||||
important to set the distributed backend to "mp" to avoid Ray scheduling
|
||||
all workers in a node other than the head node, which can cause the test
|
||||
to fail.
|
||||
"""WARNING: This test runs in both single-node (4 GPUs) and multi-node
|
||||
(2 node with 2 GPUs each) modes. If the test only uses 2 GPUs, it is
|
||||
important to set the distributed backend to "mp" to avoid Ray scheduling
|
||||
all workers in a node other than the head node, which can cause the test
|
||||
to fail.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
@@ -495,7 +495,8 @@ def test_pynccl_suspend_resume_idempotent():
|
||||
|
||||
def test_pynccl_suspend_noop_without_symbol():
|
||||
"""RCCL / NCCL < 2.29.7 lack ncclCommSuspend; suspend() must no-op, not
|
||||
crash, and resume() must then no-op too since nothing was suspended."""
|
||||
crash, and resume() must then no-op too since nothing was suspended.
|
||||
"""
|
||||
from unittest.mock import Mock
|
||||
|
||||
comm = object.__new__(PyNcclCommunicator)
|
||||
|
||||
@@ -396,8 +396,7 @@ def test_custom_quick_allreduce_bf16_cast(monkeypatch: pytest.MonkeyPatch):
|
||||
|
||||
|
||||
def qr_variable_input(rank, world_size):
|
||||
"""
|
||||
When the tensor parallelism is set to 4 or 8, frequent changes
|
||||
"""When the tensor parallelism is set to 4 or 8, frequent changes
|
||||
in the input shape can cause QuickReduce to hang (this issue
|
||||
has been observed with the gpt_oss model).
|
||||
"""
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
Integration tests for RayExecutorV2 at the executor level.
|
||||
"""Integration tests for RayExecutorV2 at the executor level.
|
||||
Validates executor initialization, placement group support, RPC calls,
|
||||
and distributed execution with various TP/PP configurations.
|
||||
"""
|
||||
@@ -299,7 +298,6 @@ def test_ray_v2_run_refs_stored_for_monitoring(executor):
|
||||
@pytest.mark.parametrize("tp_size, pp_size", [(2, 1), (2, 2)])
|
||||
def test_ray_v2_single_node_generation(tp_size, pp_size):
|
||||
"""End-to-end LLM generation with RayExecutorV2."""
|
||||
|
||||
llm = LLM(
|
||||
model=MODEL,
|
||||
tensor_parallel_size=tp_size,
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Orchestration-level integration tests for RayExecutorV2.
|
||||
"""
|
||||
"""Orchestration-level integration tests for RayExecutorV2."""
|
||||
|
||||
import gc
|
||||
import os
|
||||
@@ -25,7 +23,8 @@ def _ray_init():
|
||||
|
||||
Without this, workers cannot unpickle actor classes defined in the
|
||||
``tests`` package, causing FunctionActorManager to fall back to
|
||||
TemporaryActor which drops async method signatures."""
|
||||
TemporaryActor which drops async method signatures.
|
||||
"""
|
||||
project_root = str(pathlib.Path(__file__).resolve().parents[2])
|
||||
ray.init(
|
||||
ignore_reinit_error=True,
|
||||
@@ -158,8 +157,7 @@ def test_multi_replicas_with_bundle_indices(ray_init):
|
||||
|
||||
|
||||
def test_env_var_and_runtime_env_propagation():
|
||||
"""
|
||||
Verify env vars (NCCL_, HF_) and parallel_config.ray_runtime_env
|
||||
"""Verify env vars (NCCL_, HF_) and parallel_config.ray_runtime_env
|
||||
propagate to RayWorkerProc actors.
|
||||
"""
|
||||
sentinel_vars = {
|
||||
|
||||
@@ -53,7 +53,8 @@ META = torch.device("meta")
|
||||
|
||||
class _FakeLayer:
|
||||
"""Stand-in for a leaf module. The planner only stores it and keys on
|
||||
``id()``; nothing in the plan path touches its attributes."""
|
||||
``id()``; nothing in the plan path touches its attributes.
|
||||
"""
|
||||
|
||||
def __init__(self, tag: str) -> None:
|
||||
self.tag = tag
|
||||
@@ -64,7 +65,8 @@ class _FakeLayer:
|
||||
|
||||
def _module(layer, copies):
|
||||
"""What the bake records for one leaf module: a scatter list whose entries
|
||||
all carry that module as their ``layer``."""
|
||||
all carry that module as their ``layer``.
|
||||
"""
|
||||
for c in copies:
|
||||
c.layer = layer
|
||||
return copies
|
||||
@@ -81,7 +83,8 @@ def _copy(
|
||||
dtype=torch.bfloat16,
|
||||
):
|
||||
"""A `_Scatter` as the bake would record it: source key, owning layer, and
|
||||
the meta destination region."""
|
||||
the meta destination region.
|
||||
"""
|
||||
stride = (
|
||||
(1,)
|
||||
if len(shape) == 1
|
||||
@@ -140,7 +143,8 @@ def _planner(baked, *, name_meta, live=None, held_by=None, group_lens=None):
|
||||
|
||||
def _one_module_per_layer(n_layers, *, dtype="bfloat16", numel=4):
|
||||
"""`(baked, name_meta, names, group_lens)` for a pre / N-layer / post model
|
||||
where every gather group is one fully-baked leaf module."""
|
||||
where every gather group is one fully-baked leaf module.
|
||||
"""
|
||||
names = ["embed.weight"]
|
||||
names += [f"model.layers.{i}.w" for i in range(n_layers)]
|
||||
names += ["norm.weight"]
|
||||
@@ -161,7 +165,8 @@ def _one_module_per_layer(n_layers, *, dtype="bfloat16", numel=4):
|
||||
class TestFakeOpChains:
|
||||
"""Every allowlisted op must append itself to the chain and hand back a
|
||||
child whose shape/dtype PyTorch itself computed. The chain is the wire
|
||||
format the producer replays, so its exact contents are load-bearing."""
|
||||
format the producer replays, so its exact contents are load-bearing.
|
||||
"""
|
||||
|
||||
def _fake(self, shape=(4, 6), dtype=torch.bfloat16, sink=None):
|
||||
return FakeRDTTensor(
|
||||
@@ -222,7 +227,8 @@ class TestFakeOpChains:
|
||||
@pytest.mark.parametrize("op,n", [("chunk", 2), ("unbind", 4)])
|
||||
def test_multi_return_op_emits_one_child_per_output(self, op, n):
|
||||
"""chunk/unbind hand back a tuple; each child carries the base op plus a
|
||||
trailing __getitem__(i) so the producer can index the replayed result."""
|
||||
trailing __getitem__(i) so the producer can index the replayed result.
|
||||
"""
|
||||
parts = getattr(self._fake(), op)(*((n,) if op == "chunk" else ()), 0)
|
||||
assert isinstance(parts, tuple)
|
||||
assert len(parts) == n
|
||||
@@ -238,7 +244,8 @@ class TestFakeOpChains:
|
||||
|
||||
class TestFakeUnsupportedOps:
|
||||
"""Anything that needs real data must fail loudly at bake time rather than
|
||||
silently transferring the wrong bytes."""
|
||||
silently transferring the wrong bytes.
|
||||
"""
|
||||
|
||||
def _fake(self):
|
||||
return FakeRDTTensor(
|
||||
@@ -278,7 +285,8 @@ class TestFakeUnsupportedOps:
|
||||
|
||||
class TestBakeRecording:
|
||||
"""During the dry run the fake's ``copy_`` is the data sink: it records the
|
||||
source chain plus the meta destination's strided region and moves nothing."""
|
||||
source chain plus the meta destination's strided region and moves nothing.
|
||||
"""
|
||||
|
||||
def _recorder_and_fake(self, shape=(4, 6)):
|
||||
rec = BakeSink()
|
||||
@@ -314,7 +322,8 @@ class TestBakeRecording:
|
||||
|
||||
def test_unattributed_copy_is_live_but_unrecorded(self):
|
||||
"""A copy_ with no loader stamp cannot be attributed to a param, so its
|
||||
module must fall back to the plain load — but the name still moved data."""
|
||||
module must fall back to the plain load — but the name still moved data.
|
||||
"""
|
||||
rec, fake = self._recorder_and_fake()
|
||||
rec.current = None
|
||||
torch.empty((4, 6), dtype=torch.bfloat16, device=META).copy_(fake)
|
||||
@@ -354,7 +363,8 @@ class TestBakeRecording:
|
||||
producer packs what the replay yields, so the record must carry the
|
||||
POST-chain dtype: taking it from the source name's metadata instead sizes
|
||||
the slice with the wrong itemsize and shifts every later slice in the
|
||||
chunk, carving the packed blob differently on the two sides."""
|
||||
chunk, carving the packed blob differently on the two sides.
|
||||
"""
|
||||
rec, fake = self._recorder_and_fake(shape=(4,))
|
||||
layer = _FakeLayer("reinterpreted")
|
||||
rec.current = (layer, "weight")
|
||||
@@ -374,7 +384,8 @@ class TestBakeRecording:
|
||||
the producer packs what the chain yields, and nothing downstream can
|
||||
catch a mismatch — the consumer's buffer view is exactly prod(dest.shape)
|
||||
elements, so it reshapes cleanly over bytes laid out at other offsets.
|
||||
So the bake refuses rather than recording a slice it cannot carve."""
|
||||
So the bake refuses rather than recording a slice it cannot carve.
|
||||
"""
|
||||
rec, fake = self._recorder_and_fake(shape=(1, 6))
|
||||
rec.current = (_FakeLayer("broadcast"), "weight")
|
||||
param = torch.empty((4, 6), dtype=torch.bfloat16, device=META)
|
||||
@@ -390,7 +401,8 @@ class TestBakeRecording:
|
||||
class TestConsumerIdentity:
|
||||
"""Every worker in the fleet needs a DISTINCT id in 0..C-1: it selects the
|
||||
worker's producer block and keys the producer's per-consumer serve ring, so
|
||||
a collision silently serves two workers out of one ring."""
|
||||
a collision silently serves two workers out of one ring.
|
||||
"""
|
||||
|
||||
def _engine(self, *, dp_index, rank, world_size):
|
||||
eng = object.__new__(ShardedRDTWeightTransferEngine)
|
||||
@@ -427,7 +439,8 @@ class TestConsumerIdentity:
|
||||
def test_independent_engines_offset_into_distinct_ranges(self):
|
||||
"""A fleet of separate engines restarts _global_worker_index at 0 in
|
||||
each, so without the replica offset every engine would claim 0..w-1.
|
||||
The driver sets replica_rank on the payload; these are the ids it buys."""
|
||||
The driver sets replica_rank on the payload; these are the ids it buys.
|
||||
"""
|
||||
ids = [
|
||||
self._ids(num_consumers=8, workers=range(4), replica_rank=r, num_replicas=2)
|
||||
for r in (0, 1)
|
||||
@@ -453,7 +466,8 @@ def _router(
|
||||
group_lens=None,
|
||||
):
|
||||
"""A router over ``n`` single-name groups unless told otherwise, so a test
|
||||
can state expectations in group indices."""
|
||||
can state expectations in group indices.
|
||||
"""
|
||||
if names is None:
|
||||
n = len(name_owner_class or []) or (len(group_lens or []) or 1)
|
||||
names = [f"g{i}" for i in range(n)]
|
||||
@@ -482,7 +496,8 @@ class TestRdtRouter:
|
||||
def test_every_producer_nic_carries_traffic(self):
|
||||
"""16 producers / 8 consumers: the block rule spreads each consumer's
|
||||
pulls over its block, alternating by group, so no producer sits idle.
|
||||
Plain `consumer_id % len(owners)` would idle half of them."""
|
||||
Plain `consumer_id % len(owners)` would idle half of them.
|
||||
"""
|
||||
r = _router(16, 8, group_lens=[1] * 95)
|
||||
names = [f"g{i}" for i in range(95)]
|
||||
for c in range(8):
|
||||
@@ -500,7 +515,8 @@ class TestRdtRouter:
|
||||
|
||||
def test_pipeline_stages_route_to_the_owning_stage(self):
|
||||
"""2 stages x 8 ranks: groups 0-2 on stage 0, 3-5 on stage 1. Two owner
|
||||
sets, one per stage; each consumer reaches both."""
|
||||
sets, one per stage; each consumer reaches both.
|
||||
"""
|
||||
r = _router(
|
||||
16,
|
||||
8,
|
||||
@@ -516,7 +532,8 @@ class TestRdtRouter:
|
||||
|
||||
def test_expert_names_route_to_the_holding_ranks(self):
|
||||
"""A name held by only some ranks routes to those; a replicated name
|
||||
keeps the full set."""
|
||||
keeps the full set.
|
||||
"""
|
||||
r = RdtRouter(
|
||||
8,
|
||||
4,
|
||||
@@ -535,7 +552,8 @@ class TestRdtRouter:
|
||||
|
||||
def test_consumers_spread_over_an_owner_set(self):
|
||||
"""Several ranks hold a name (its TP peers); the block rule must spread
|
||||
consumers across them, not funnel through one."""
|
||||
consumers across them, not funnel through one.
|
||||
"""
|
||||
r = _router(
|
||||
4, 4, owner_sets=[[0, 1]], name_owner_class=[0] * 6, group_lens=[1] * 6
|
||||
)
|
||||
@@ -544,7 +562,8 @@ class TestRdtRouter:
|
||||
|
||||
def test_a_group_frees_at_every_owner_of_any_of_its_names(self):
|
||||
"""The free barrier is per group, so its fan-out is the UNION over the
|
||||
group's names: a rank holding none of them is not signalled."""
|
||||
group's names: a rank holding none of them is not signalled.
|
||||
"""
|
||||
r = RdtRouter(4, 2, [[0, 1], [2, 3]], [0, 1, 0], ["a", "b", "c"], [2, 1])
|
||||
assert r.group_owners(0) == [0, 1, 2, 3]
|
||||
assert r.group_owners(1) == [0, 1]
|
||||
@@ -614,7 +633,8 @@ class TestReplicaOverlay:
|
||||
|
||||
def test_adding_a_deployment_does_not_move_the_first_one(self):
|
||||
"""The strongest form: an existing consumer's routes are byte-identical
|
||||
before and after another deployment is provisioned."""
|
||||
before and after another deployment is provisioned.
|
||||
"""
|
||||
one, names = self._fleet(16, 8, 1)
|
||||
two, _ = self._fleet(16, 8, 2)
|
||||
for consumer_id in range(8):
|
||||
@@ -624,7 +644,8 @@ class TestReplicaOverlay:
|
||||
|
||||
def test_one_deployment_is_the_historical_carve(self):
|
||||
"""Default (no width) and width == the whole fleet must agree with the
|
||||
plain block rule, so a single-deployment fleet is untouched."""
|
||||
plain block rule, so a single-deployment fleet is untouched.
|
||||
"""
|
||||
plain = _router(16, 8, group_lens=[1] * 6)
|
||||
explicit, names = self._fleet(16, 8, 1)
|
||||
for consumer_id in range(8):
|
||||
@@ -638,7 +659,8 @@ class TestReplicaOverlay:
|
||||
def test_the_overlay_holds_for_partially_owned_names(self):
|
||||
"""Pipeline stages and experts too: two owner sets, one narrow. The
|
||||
narrow one already forced agreement (one owner, no block to carve); the
|
||||
wide one is what the carve decides."""
|
||||
wide one is what the carve decides.
|
||||
"""
|
||||
r, names = self._fleet(
|
||||
8,
|
||||
4,
|
||||
@@ -654,14 +676,16 @@ class TestReplicaOverlay:
|
||||
def test_a_width_that_does_not_divide_the_fleet_raises(self):
|
||||
"""A uniform fleet is asserted upstream in ``get_world_size``; if it ever
|
||||
is not, the overlay would map two workers of one deployment onto the same
|
||||
block index, so refuse rather than serve the wrong bytes."""
|
||||
block index, so refuse rather than serve the wrong bytes.
|
||||
"""
|
||||
with pytest.raises(ValueError, match="does not divide"):
|
||||
RdtRouter(4, 6, None, None, ["a"], [1], workers_per_replica=4)
|
||||
|
||||
def test_pulls_still_carry_the_global_consumer_id(self):
|
||||
"""The carve uses the index WITHIN a deployment, but the wire must keep
|
||||
the fleet-global id: it is what lets the producer tell the sharers of a
|
||||
slot apart, and count their arrivals separately."""
|
||||
slot apart, and count their arrivals separately.
|
||||
"""
|
||||
|
||||
class _Method:
|
||||
def __init__(self):
|
||||
@@ -704,7 +728,8 @@ def _producer_pack_offsets(slices):
|
||||
class TestPackedLayout:
|
||||
def test_consumer_layout_matches_the_producer_rule(self):
|
||||
"""Mixed dtypes, sizes that do not land on 16B — i.e. the Kimi-style
|
||||
group (fp8 weights + fp32 scales + bf16 norms) that exposes alignment."""
|
||||
group (fp8 weights + fp32 scales + bf16 norms) that exposes alignment.
|
||||
"""
|
||||
specs = [
|
||||
("w.fp8", "float8_e4m3fn", [17]),
|
||||
("w.scale", "float32", [3]),
|
||||
@@ -779,7 +804,8 @@ class TestPackedLayout:
|
||||
class TestChunkModuleScatters:
|
||||
"""The chunk cut: one chunk per distinct owner class present in the copies,
|
||||
ascending by class index. Derived purely from the bake plus the ownership
|
||||
table, so vLLM's expert placement needs no cases."""
|
||||
table, so vLLM's expert placement needs no cases.
|
||||
"""
|
||||
|
||||
def test_unstamped_copies_form_one_chunk(self):
|
||||
layer = _FakeLayer("l")
|
||||
@@ -793,7 +819,8 @@ class TestChunkModuleScatters:
|
||||
"""Classes are numbered by first appearance in metadata order, so the
|
||||
replicated class (0) leads and the rest follow — the ordering the old
|
||||
"-1 first, then ascending stamp" rule produced, now falling out of the
|
||||
table instead of a special case."""
|
||||
table instead of a special case.
|
||||
"""
|
||||
layer = _FakeLayer("moe")
|
||||
names = ["norm", "e5", "e0", "e2"]
|
||||
copies = [_copy(n, shape=(4,)) for n in names]
|
||||
@@ -813,7 +840,8 @@ class TestChunkModuleScatters:
|
||||
|
||||
def test_copy_order_within_a_chunk_is_bake_order(self):
|
||||
"""Scatter order within a chunk must follow the bake, not the stamp
|
||||
sort: the packed layout and the producer's replay agree on keys order."""
|
||||
sort: the packed layout and the producer's replay agree on keys order.
|
||||
"""
|
||||
layer = _FakeLayer("moe")
|
||||
names = ["e3", "e1", "e2"] # one stamp, deliberately unsorted names
|
||||
copies = [_copy(n, shape=(4,)) for n in names]
|
||||
@@ -827,7 +855,8 @@ class TestChunkModuleScatters:
|
||||
|
||||
def test_stamps_interleave_across_modules_without_reordering_within(self):
|
||||
"""Two modules' copies bucket by stamp; within each bucket the order is
|
||||
module-then-bake order."""
|
||||
module-then-bake order.
|
||||
"""
|
||||
a, b = _FakeLayer("a"), _FakeLayer("b")
|
||||
mods = [
|
||||
_module(a, [_copy("a0", shape=(4,)), _copy("a1", shape=(4,))]),
|
||||
@@ -845,9 +874,10 @@ class TestChunkModuleScatters:
|
||||
]
|
||||
|
||||
def test_scatters_carry_their_own_dtype(self):
|
||||
"""dtype rides the record from the bake, where it is the fake's dtype
|
||||
"""Dtype rides the record from the bake, where it is the fake's dtype
|
||||
AFTER its op chain — not a plan-time lookup of the source name, which
|
||||
would be wrong for any chain that reinterprets dtype."""
|
||||
would be wrong for any chain that reinterprets dtype.
|
||||
"""
|
||||
layer = _FakeLayer("l")
|
||||
eng = _planner({}, name_meta={"w": ("float32", [10])})
|
||||
((_er, scatters),) = eng._chunk_module_scatters(
|
||||
@@ -886,7 +916,8 @@ class TestBuildCallPlan:
|
||||
|
||||
def test_materialize_fires_on_a_modules_first_chunk_only(self):
|
||||
"""Empty HF params are allocated once per module, by construction —
|
||||
including a FusedMoE-like module whose copies span owner-class chunks."""
|
||||
including a FusedMoE-like module whose copies span owner-class chunks.
|
||||
"""
|
||||
layer = _FakeLayer("spanning")
|
||||
names = [f"w{i}" for i in range(4)]
|
||||
copies = [_copy(n, shape=(4,)) for n in names]
|
||||
@@ -921,7 +952,8 @@ class TestBuildCallPlan:
|
||||
def test_free_signal_waits_for_a_groups_last_chunk_when_it_is_split(self):
|
||||
"""The load-bearing case: with the group cut across owner-class chunks the
|
||||
signal must hang off the LAST one. Signaling earlier lets the producers
|
||||
drop the gather buffers while a later chunk's RDMA is still reading."""
|
||||
drop the gather buffers while a later chunk's RDMA is still reading.
|
||||
"""
|
||||
layer = _FakeLayer("l")
|
||||
names = [f"w{i}" for i in range(4)]
|
||||
copies = [_copy(n, shape=(4,)) for n in names]
|
||||
@@ -937,7 +969,8 @@ class TestBuildCallPlan:
|
||||
|
||||
def test_free_signal_timing_across_two_split_groups(self):
|
||||
"""Two groups x 2 owner-class chunks each: signals land on each group's
|
||||
last chunk only."""
|
||||
last chunk only.
|
||||
"""
|
||||
names_a = [f"model.layers.0.w{i}" for i in range(4)]
|
||||
names_b = [f"model.layers.1.w{i}" for i in range(4)]
|
||||
baked, name_meta, stamps = {}, {}, {}
|
||||
@@ -955,7 +988,8 @@ class TestBuildCallPlan:
|
||||
def test_a_group_with_nothing_local_is_signaled_at_sync_start(self):
|
||||
"""The owners still published it, so this consumer must signal it —
|
||||
immediately, before the pipeline (signal-before-publish is tolerated),
|
||||
never hung off another group's chunk."""
|
||||
never hung off another group's chunk.
|
||||
"""
|
||||
baked, name_meta, names, group_lens = _one_module_per_layer(2)
|
||||
del baked["model.layers.1.w"] # nothing baked, and not live => no pull
|
||||
eng = _planner(
|
||||
@@ -976,7 +1010,8 @@ class TestBuildCallPlan:
|
||||
|
||||
def test_modules_are_deduped_within_a_group(self):
|
||||
"""Several names of one fused module map to the same scatter list; the
|
||||
plan must not chunk it twice."""
|
||||
plan must not chunk it twice.
|
||||
"""
|
||||
layer = _FakeLayer("fused")
|
||||
names = ["qkv.q", "qkv.k", "qkv.v"]
|
||||
copies = [_copy(n, shape=(4,)) for n in names]
|
||||
@@ -992,7 +1027,8 @@ class TestBuildCallPlan:
|
||||
|
||||
def test_plan_is_a_pure_function_of_its_inputs(self):
|
||||
"""Called twice with the same arguments it must produce the same plan —
|
||||
that is what lets the engine build it once and reuse it every sync."""
|
||||
that is what lets the engine build it once and reuse it every sync.
|
||||
"""
|
||||
baked, name_meta, names, group_lens = _one_module_per_layer(2)
|
||||
eng = _planner(baked, name_meta=name_meta)
|
||||
a = eng._build_call_plan(names, group_lens)
|
||||
@@ -1012,7 +1048,8 @@ class TestBuildCallPlan:
|
||||
class TestUnbakedNamesGuard:
|
||||
"""A live name with no baked plan has no way to load — there is no fallback,
|
||||
and its pull would target groups the pipeline already freed — so the plan
|
||||
build fails at init, naming the names."""
|
||||
build fails at init, naming the names.
|
||||
"""
|
||||
|
||||
def test_a_live_unbaked_name_fails_the_plan_build(self):
|
||||
baked, name_meta, names, group_lens = _one_module_per_layer(2)
|
||||
@@ -1023,7 +1060,8 @@ class TestUnbakedNamesGuard:
|
||||
|
||||
def test_never_copied_names_are_dropped_entirely(self):
|
||||
"""Experts owned by another EP rank no-op in their loader: not an error,
|
||||
and not a chunk — their group is pre-freed if nothing else fills it."""
|
||||
and not a chunk — their group is pre-freed if nothing else fills it.
|
||||
"""
|
||||
baked, name_meta, names, group_lens = _one_module_per_layer(2)
|
||||
del baked["norm.weight"]
|
||||
eng = _planner(baked, name_meta=name_meta, live=set(names) - {"norm.weight"})
|
||||
@@ -1035,13 +1073,15 @@ class TestUnbakedNamesGuard:
|
||||
class TestCallPlanRouting:
|
||||
"""Under partial ownership each chunk must be pulled from a producer that
|
||||
actually holds every name in it. Consumers bind EVERY producer (signals fan
|
||||
out to all owners), so local index == trainer rank."""
|
||||
out to all owners), so local index == trainer rank.
|
||||
"""
|
||||
|
||||
def _routed_planner(
|
||||
self, held_by_group, num_consumers, consumer_id, *, held_by=None
|
||||
):
|
||||
"""``held_by_group[g]`` = the ranks holding group g's names, so a test
|
||||
can still speak in groups; ``held_by`` overrides individual names."""
|
||||
can still speak in groups; ``held_by`` overrides individual names.
|
||||
"""
|
||||
baked, name_meta, names, group_lens = _one_module_per_layer(
|
||||
len(held_by_group) - 2
|
||||
)
|
||||
@@ -1068,14 +1108,16 @@ class TestCallPlanRouting:
|
||||
|
||||
def test_two_stage_ownership_splits_chunks_between_owners(self):
|
||||
"""Two PP stages, 4 groups: stage 0 holds the first half, stage 1 the
|
||||
second. Every chunk goes to a holder of its names."""
|
||||
second. Every chunk goes to a holder of its names.
|
||||
"""
|
||||
eng, names, group_lens = self._routed_planner([[0], [0], [1], [1]], 1, 0)
|
||||
plan = eng._build_call_plan(names, group_lens)
|
||||
assert [c.owner for c in plan.chunks] == [0, 0, 1, 1]
|
||||
|
||||
def test_a_multi_owner_group_still_resolves_to_one_producer(self):
|
||||
"""Every chunk is served by exactly ONE producer: splitting a pull only
|
||||
multiplies produce calls, since the consumer's NIC bounds it."""
|
||||
multiplies produce calls, since the consumer's NIC bounds it.
|
||||
"""
|
||||
eng, names, group_lens = self._routed_planner([[0, 1]] * 4, 1, 0)
|
||||
plan = eng._build_call_plan(names, group_lens)
|
||||
assert all(0 <= c.owner < 2 for c in plan.chunks)
|
||||
@@ -1083,7 +1125,8 @@ class TestCallPlanRouting:
|
||||
def test_an_expert_name_routes_to_the_rank_holding_it(self):
|
||||
"""PP ∩ EP: stage 0 = ranks 0-1, stage 1 = ranks 2-3, and one expert
|
||||
name inside a stage-1 group is held by rank 3 alone. The name's own owner
|
||||
set is what routes it."""
|
||||
set is what routes it.
|
||||
"""
|
||||
eng, names, group_lens = self._routed_planner(
|
||||
[[0, 1], [0, 1], [2, 3], [2, 3]],
|
||||
1,
|
||||
@@ -1098,7 +1141,8 @@ class TestCallPlanRouting:
|
||||
def test_an_unheld_name_is_rejected_at_init(self):
|
||||
"""A name no rank holds can never be served. The trainer raises when it
|
||||
transposes the holdings; the router's validate() is the consumer-side
|
||||
backstop, and either way it fails at init rather than at first pull."""
|
||||
backstop, and either way it fails at init rather than at first pull.
|
||||
"""
|
||||
with pytest.raises(ValueError, match="empty"):
|
||||
RdtRouter(2, 1, [[]], [0], ["w"], [1]).validate()
|
||||
|
||||
@@ -1107,14 +1151,16 @@ class TestSignalCompleteness:
|
||||
"""Every gather group is signaled exactly once — after its last chunk when
|
||||
the worker pulls from it, at sync start otherwise — for ANY expert
|
||||
placement. A group signaled twice over-credits the barrier; a group never
|
||||
signaled parks a producer credit and hangs end_sync."""
|
||||
signaled parks a producer credit and hangs end_sync.
|
||||
"""
|
||||
|
||||
def _moe_planner(self, worker_experts, *, ep_size=4, n_experts=8, n_layers=2):
|
||||
"""pre / n_layers MoE layers / post. Each layer: one norm (its own
|
||||
"""Pre / n_layers MoE layers / post. Each layer: one norm (its own
|
||||
module, stamp -1) + n_experts expert names stamped ``e // n_local``.
|
||||
The worker's bake covers only ``worker_experts`` (its placement);
|
||||
foreign expert names never copied => dropped entirely, as in the real
|
||||
bake."""
|
||||
bake.
|
||||
"""
|
||||
n_local = n_experts // ep_size
|
||||
names = ["embed.weight"]
|
||||
group_lens = [1]
|
||||
@@ -1173,8 +1219,9 @@ class TestSignalCompleteness:
|
||||
assert self._signals(plan) == list(range(len(group_lens)))
|
||||
|
||||
def test_placement_changes_only_the_chunk_count_never_the_signals(self):
|
||||
"""linear experts 0-3 hit 2 owner classes; round_robin 0,2,4,6
|
||||
hits all 4. More chunks per group, identical signal set."""
|
||||
"""Linear experts 0-3 hit 2 owner classes; round_robin 0,2,4,6
|
||||
hits all 4. More chunks per group, identical signal set.
|
||||
"""
|
||||
plans = {}
|
||||
for label, experts in (("linear", [0, 1, 2, 3]), ("round_robin", [0, 2, 4, 6])):
|
||||
eng, names, group_lens = self._moe_planner(worker_experts=experts)
|
||||
@@ -1188,7 +1235,8 @@ class TestSignalCompleteness:
|
||||
|
||||
def test_the_fused_module_materializes_first_and_quants_last_across_chunks(self):
|
||||
"""The FusedMoE shape: one module's copies span every owner-class chunk of
|
||||
its group; materialize on its first, quant on its last."""
|
||||
its group; materialize on its first, quant on its last.
|
||||
"""
|
||||
eng, names, group_lens = self._moe_planner(
|
||||
worker_experts=[0, 2, 4, 6], n_layers=1
|
||||
)
|
||||
@@ -1246,7 +1294,8 @@ class TestLayerwiseGroups:
|
||||
def test_flattening_a_partition_returns_the_input_order(self):
|
||||
"""Un-indexed names group by POSITION, not by name class — so one that
|
||||
looks like a pre-block name still lands after the layers it follows.
|
||||
Group index therefore means the same thing on every rank."""
|
||||
Group index therefore means the same thing on every rank.
|
||||
"""
|
||||
names = [
|
||||
"embed.weight",
|
||||
"model.layers.0.a",
|
||||
@@ -1264,7 +1313,8 @@ class TestLayerwiseGroups:
|
||||
|
||||
def test_stacks_keep_prefix_appearance_order_and_sort_within(self):
|
||||
"""Two stacks stay in the order they appear — the vision tower before the
|
||||
text stack — while each sorts by index internally."""
|
||||
text stack — while each sorts by index internally.
|
||||
"""
|
||||
names = [
|
||||
"visual.blocks.1.w",
|
||||
"visual.blocks.0.w",
|
||||
@@ -1281,7 +1331,8 @@ class TestLayerwiseGroups:
|
||||
def test_the_post_block_lands_last_however_early_it_arrives(self):
|
||||
"""Megatron-Bridge streams the last pipeline stage's output block before
|
||||
its layers. Sweeping un-indexed names after the first layer into a
|
||||
trailing group is what lets the gather loop walk groups ascending."""
|
||||
trailing group is what lets the gather loop walk groups ascending.
|
||||
"""
|
||||
names = ["model.layers.0.w", "model.norm.weight", "model.layers.1.w"]
|
||||
assert layerwise_groups(names) == [
|
||||
["model.layers.0.w"],
|
||||
@@ -1291,7 +1342,8 @@ class TestLayerwiseGroups:
|
||||
|
||||
def test_names_sharing_a_layer_coalesce_when_the_source_interleaves(self):
|
||||
"""A raw-checkpoint source may yield in shard-packing order. Coalescing on
|
||||
the key keeps that from shattering one layer into many groups."""
|
||||
the key keeps that from shattering one layer into many groups.
|
||||
"""
|
||||
names = [
|
||||
"model.layers.0.a",
|
||||
"model.layers.1.a",
|
||||
@@ -1333,13 +1385,15 @@ class TestLayerwiseGroups:
|
||||
def test_every_supported_naming_convention_partitions_per_layer(self, prefix):
|
||||
"""One decoder layer per group under any prefix, with no per-architecture
|
||||
table. A literal `model.layers.` match silently yields one whole-model
|
||||
group for most of these."""
|
||||
group for most of these.
|
||||
"""
|
||||
names = [f"{prefix}{i}.self_attn.q_proj.weight" for i in range(3)]
|
||||
assert layerwise_groups(names) == [[n] for n in names]
|
||||
|
||||
def test_a_moe_layer_stays_whole(self):
|
||||
"""The OUTERMOST index wins, so per-expert names group by layer. Splitting
|
||||
on the expert would cut FusedMoE — one leaf module — across groups."""
|
||||
on the expert would cut FusedMoE — one leaf module — across groups.
|
||||
"""
|
||||
names = [
|
||||
"model.layers.0.mlp.experts.0.w1.weight",
|
||||
"model.layers.0.mlp.experts.1.w1.weight",
|
||||
@@ -1356,7 +1410,8 @@ class TestLayerwiseGroups:
|
||||
def test_a_vlm_partitions_the_vision_tower_and_the_text_stack(self):
|
||||
"""Qwen-VL layout. Today's literal `model.layers.` match puts every one of
|
||||
these in ONE group — the whole model — because the text stack ships as
|
||||
`model.language_model.layers.`."""
|
||||
`model.language_model.layers.`.
|
||||
"""
|
||||
names = [
|
||||
"visual.patch_embed.proj.weight",
|
||||
"visual.blocks.0.attn.qkv.weight",
|
||||
@@ -1375,7 +1430,8 @@ class TestLayerwiseGroups:
|
||||
|
||||
def test_two_stacks_sharing_an_index_do_not_merge(self):
|
||||
"""The key carries the prefix, so vision block 0 and text layer 0 are
|
||||
different groups even though both are index 0."""
|
||||
different groups even though both are index 0.
|
||||
"""
|
||||
names = [
|
||||
"visual.blocks.0.attn.qkv.weight",
|
||||
"model.language_model.layers.0.self_attn.q_proj.weight",
|
||||
@@ -1412,20 +1468,23 @@ class TestOpAllowlistAgreement:
|
||||
|
||||
def test_every_bake_recordable_op_is_replayable(self):
|
||||
"""Anything the bake can record must be serveable, or the failure lands a
|
||||
whole sync later than the mistake."""
|
||||
whole sync later than the mistake.
|
||||
"""
|
||||
assert not set(SUPPORTED_OPS.values()) - set(ALLOWED_OPS)
|
||||
|
||||
def test_the_producer_allows_nothing_the_consumer_cannot_emit(self):
|
||||
"""The allowlist is also a guard against a misbehaving or spoofed
|
||||
consumer invoking arbitrary methods on trainer tensors, so it must not be
|
||||
wider than the ops the bake can actually produce. ``to`` in particular
|
||||
would let a replay change dtype or device."""
|
||||
would let a replay change dtype or device.
|
||||
"""
|
||||
assert not set(ALLOWED_OPS) - set(SUPPORTED_OPS.values())
|
||||
assert "to" not in ALLOWED_OPS
|
||||
|
||||
def test_transpose_via_t_is_serveable(self):
|
||||
"""The specific regression: a chain recorded from ``.t()`` must pass the
|
||||
producer's guard."""
|
||||
producer's guard.
|
||||
"""
|
||||
fake = FakeRDTTensor(
|
||||
name="w",
|
||||
shape=torch.Size((4, 6)),
|
||||
@@ -1470,7 +1529,8 @@ BAKE_MODELS = [
|
||||
class TestRequiresTheRayExecutor:
|
||||
"""The engine's data plane is Ray's, so a non-Ray executor cannot work. The
|
||||
check is at construction because the alternative is an opaque failure during
|
||||
the first handshake, long after the misconfiguration."""
|
||||
the first handshake, long after the misconfiguration.
|
||||
"""
|
||||
|
||||
def _construct(self, backend):
|
||||
cfg = SimpleNamespace(
|
||||
@@ -1497,7 +1557,8 @@ class TestRequiresTheRayExecutor:
|
||||
|
||||
def test_a_custom_executor_class_is_left_alone(self):
|
||||
"""A `type[Executor]` override is deliberate and unjudgeable here, so it
|
||||
must not be rejected for merely not being the string "ray"."""
|
||||
must not be rejected for merely not being the string "ray".
|
||||
"""
|
||||
|
||||
class _CustomExecutor:
|
||||
pass
|
||||
|
||||
@@ -236,7 +236,8 @@ class TestPublishAndRebuild:
|
||||
def test_views_of_one_storage_do_not_overlap(self, server_factory):
|
||||
"""One IPC export per storage, one as_strided view per name — the whole
|
||||
point of the storage/view split. Overlapping views would serve the same
|
||||
bytes for two different weights."""
|
||||
bytes for two different weights.
|
||||
"""
|
||||
server = server_factory()
|
||||
server.begin_sync(1)
|
||||
_publish(server, GI_A, GROUP_A, nbytes=64)
|
||||
@@ -259,7 +260,8 @@ class TestPublishAndRebuild:
|
||||
def test_the_default_lookahead_is_one(self):
|
||||
"""1 is the sweet spot: group N+1 is gathered AND pullable while N is
|
||||
being pulled, with resident memory at its floor of 2 groups — the bound
|
||||
the larger-model runs size against."""
|
||||
the larger-model runs size against.
|
||||
"""
|
||||
assert DEFAULT_GATHER_LOOKAHEAD == 1
|
||||
|
||||
|
||||
@@ -274,7 +276,8 @@ class TestFreeBarrier:
|
||||
|
||||
def test_group_is_held_until_the_last_live_consumer_signals(self, server_factory):
|
||||
"""Every live consumer signals every owner; releasing on the first
|
||||
signal would drop storage other consumers are still reading."""
|
||||
signal would drop storage other consumers are still reading.
|
||||
"""
|
||||
server = server_factory()
|
||||
server.begin_sync(3)
|
||||
_publish(server, GI_A, GROUP_A)
|
||||
@@ -301,7 +304,8 @@ class TestFreeBarrier:
|
||||
):
|
||||
"""A consumer with nothing to pull from a group signals it at sync
|
||||
start, which can precede the publish. The publish must then release the
|
||||
group rather than wait for a signal that will never come again."""
|
||||
group rather than wait for a signal that will never come again.
|
||||
"""
|
||||
server = server_factory()
|
||||
server.begin_sync(1)
|
||||
server.free_group(GI_A)
|
||||
@@ -324,7 +328,8 @@ class TestFreeBarrier:
|
||||
the server says the group is gone — and it hears that ONLY through
|
||||
``wait_freed`` / ``end_sync``, never a publish return (a freed notice
|
||||
riding an unharvested async publish while the engine blocks in
|
||||
``wait_freed`` would wedge the loop)."""
|
||||
``wait_freed`` would wedge the loop).
|
||||
"""
|
||||
server = server_factory()
|
||||
server.begin_sync(1)
|
||||
_publish(server, GI_A, GROUP_A)
|
||||
@@ -353,7 +358,8 @@ class TestFreeBarrier:
|
||||
|
||||
class _CountingSource:
|
||||
"""`iter_groups` yielding one CPU-tensor group per call, built lazily so a
|
||||
group's tensors exist only once the engine's credit gate lets it gather."""
|
||||
group's tensors exist only once the engine's credit gate lets it gather.
|
||||
"""
|
||||
|
||||
def __init__(self, groups):
|
||||
self._groups = groups
|
||||
@@ -366,7 +372,8 @@ class _CountingSource:
|
||||
class _ResidencyDict(dict):
|
||||
"""The engine's `_inflight`, instrumented: `max_len` is the high-water mark
|
||||
of groups whose CUDA-IPC export refs were alive at once — the trainer's
|
||||
resident-memory bound in groups."""
|
||||
resident-memory bound in groups.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
@@ -380,7 +387,8 @@ class _ResidencyDict(dict):
|
||||
def _loop_engine(server, n_groups, *, lookahead):
|
||||
"""A real engine wired straight to a real (non-Ray) server, with only the
|
||||
state `_run_gather_loop` reads. The `_rpc` seam dispatches inline;
|
||||
`_publish_async` sees no `.remote` and runs inline too."""
|
||||
`_publish_async` sees no `.remote` and runs inline too.
|
||||
"""
|
||||
groups = [(f"model.layers.{gi}.w",) for gi in range(n_groups)]
|
||||
e = ShardedRDTTrainerWeightTransferEngine.__new__(
|
||||
ShardedRDTTrainerWeightTransferEngine
|
||||
@@ -403,14 +411,16 @@ class TestGatherCredit:
|
||||
serveable immediately, and the ENGINE's loop stops gathering while more than
|
||||
`gather_lookahead` groups are unfreed. So at most `lookahead + 1` groups are
|
||||
resident — two at the default — while group N+1 is pulled the instant N's
|
||||
pulls finish."""
|
||||
pulls finish.
|
||||
"""
|
||||
|
||||
@pytest.fixture
|
||||
def gather_engine(self, monkeypatch):
|
||||
"""Neutralize the two CUDA touches in the export path so the REAL
|
||||
gather loop runs on CPU: `.cuda()` becomes identity, and the storage
|
||||
export ships the `(nbytes, ..., device, ...)` tuple the fixture's
|
||||
stubbed `rebuild_cuda_tensor` expects."""
|
||||
stubbed `rebuild_cuda_tensor` expects.
|
||||
"""
|
||||
monkeypatch.setattr(torch.Tensor, "cuda", lambda self: self)
|
||||
monkeypatch.setattr(
|
||||
trainer_mod,
|
||||
@@ -422,7 +432,8 @@ class TestGatherCredit:
|
||||
def test_publish_never_blocks(self, server_factory):
|
||||
"""Publishing must not park on a credit: the whole point is that a
|
||||
gathered group becomes serveable immediately. Three publishes with no
|
||||
frees, inline on this thread — a block here hangs the test."""
|
||||
frees, inline on this thread — a block here hangs the test.
|
||||
"""
|
||||
server = server_factory(gather_lookahead=1)
|
||||
server.begin_sync(1)
|
||||
for gi in range(3):
|
||||
@@ -436,7 +447,8 @@ class TestGatherCredit:
|
||||
than 2 groups of gathered weights, no matter how the consumer paces —
|
||||
and the pipeline still overlaps, because the consumer here refuses to
|
||||
free group N until N+1 is already published (pullable). A gate an
|
||||
off-by-one too tight deadlocks this test; too loose fails max_len."""
|
||||
off-by-one too tight deadlocks this test; too loose fails max_len.
|
||||
"""
|
||||
server = server_factory(gather_lookahead=1)
|
||||
n_groups = 8
|
||||
engine = gather_engine(server, n_groups, lookahead=1)
|
||||
@@ -478,7 +490,8 @@ class TestGatherCredit:
|
||||
):
|
||||
"""0 is the serialized baseline: the loop waits for group N to be freed
|
||||
before gathering N+1, so exactly ONE group is ever resident and nothing
|
||||
overlaps. A floor of 1 on the bound would silently pipeline instead."""
|
||||
overlaps. A floor of 1 on the bound would silently pipeline instead.
|
||||
"""
|
||||
server = server_factory(gather_lookahead=0)
|
||||
engine = gather_engine(server, 4, lookahead=0)
|
||||
|
||||
@@ -503,8 +516,9 @@ class TestGatherCredit:
|
||||
def test_the_residency_bound_scales_with_the_lookahead(
|
||||
self, server_factory, gather_engine
|
||||
):
|
||||
"""lookahead + 1, not a hardcoded 2: at lookahead=2 a free-nothing
|
||||
consumer sees exactly 3 groups gathered before the loop parks."""
|
||||
"""Lookahead + 1, not a hardcoded 2: at lookahead=2 a free-nothing
|
||||
consumer sees exactly 3 groups gathered before the loop parks.
|
||||
"""
|
||||
server = server_factory(gather_lookahead=2)
|
||||
engine = gather_engine(server, 8, lookahead=2)
|
||||
|
||||
@@ -540,7 +554,8 @@ class TestGatherCredit:
|
||||
def test_a_gather_error_releases_a_blocked_wait_freed(self, server_factory):
|
||||
"""Otherwise a trainer-side failure on another rank deadlocks this one
|
||||
inside its credit gate. It must RAISE, not return empty — an empty
|
||||
return would spin the engine straight back into the wait."""
|
||||
return would spin the engine straight back into the wait.
|
||||
"""
|
||||
server = server_factory()
|
||||
server.begin_sync(1)
|
||||
_publish(server, GI_A, GROUP_A)
|
||||
@@ -621,7 +636,8 @@ class TestBeginSync:
|
||||
|
||||
def test_the_target_is_floored_at_one(self, server_factory):
|
||||
"""A bare/zero call must not make every group free instantly (or divide
|
||||
the barrier by zero)."""
|
||||
the barrier by zero).
|
||||
"""
|
||||
server = server_factory()
|
||||
server.begin_sync(0)
|
||||
assert server._live_count == 1
|
||||
@@ -629,7 +645,8 @@ class TestBeginSync:
|
||||
def test_a_degraded_sync_lowers_the_target(self, server_factory):
|
||||
"""FT degraded sync: the live count is the WHOLE degraded-sync
|
||||
mechanism on this side — the group releases after the live consumers
|
||||
alone."""
|
||||
alone.
|
||||
"""
|
||||
server = server_factory()
|
||||
server.begin_sync(2) # 4 provisioned, 2 alive
|
||||
_publish(server, GI_A, GROUP_A)
|
||||
@@ -653,7 +670,8 @@ class TestBeginSync:
|
||||
def test_a_straggler_signal_cannot_credit_the_next_sync(self, server_factory):
|
||||
"""begin_sync resets the counts, so a signal that leaked past the sync
|
||||
boundary would over-credit — the reason the consumer drains its fired
|
||||
signals before finishing. Pin the reset side of that contract."""
|
||||
signals before finishing. Pin the reset side of that contract.
|
||||
"""
|
||||
server = server_factory()
|
||||
server.begin_sync(2)
|
||||
_publish(server, GI_A, GROUP_A)
|
||||
@@ -675,7 +693,8 @@ class TestBeginSync:
|
||||
self, server_factory
|
||||
):
|
||||
"""The packed layout repeats every sync — that is what makes caching the
|
||||
destination views worth ~5ms per 384-spec group."""
|
||||
destination views worth ~5ms per 384-spec group.
|
||||
"""
|
||||
server = server_factory()
|
||||
server._pack_dsts[("sentinel",)] = (0, [])
|
||||
server.begin_sync(1)
|
||||
@@ -686,7 +705,8 @@ class TestServedNamesGuard:
|
||||
"""A producer serves only the names it holds: its stage's groups, and
|
||||
within them its own EP coordinate's experts plus the replicated names.
|
||||
Without this guard a misrouted pull would block forever in the cache wait
|
||||
for a name this rank never publishes."""
|
||||
for a name this rank never publishes.
|
||||
"""
|
||||
|
||||
def test_a_pull_for_an_unserved_name_fails_loudly(self, server_factory):
|
||||
server = server_factory(served_names=list(GROUP_A))
|
||||
@@ -702,7 +722,8 @@ class TestServedNamesGuard:
|
||||
|
||||
def test_served_names_none_accepts_anything(self, server_factory):
|
||||
"""Gather-to-all: every producer holds every group, so there is nothing
|
||||
to guard. The call proceeds to the cache wait (not exercised here)."""
|
||||
to guard. The call proceeds to the cache wait (not exercised here).
|
||||
"""
|
||||
server = server_factory(served_names=None)
|
||||
assert server._served_names is None
|
||||
|
||||
@@ -749,7 +770,8 @@ class TestStallWatchdog:
|
||||
|
||||
def test_end_sync_fails_when_a_consumer_never_signals(self, server_factory):
|
||||
"""The exact mid-sync death signature: the group is published, one of
|
||||
the two live consumers that owed a signal is gone."""
|
||||
the two live consumers that owed a signal is gone.
|
||||
"""
|
||||
server = server_factory(stall_timeout_s=0.3)
|
||||
server.begin_sync(2)
|
||||
_publish(server, GI_A, GROUP_A)
|
||||
@@ -763,7 +785,8 @@ class TestStallWatchdog:
|
||||
"""The watchdog fires on the existing `set_gather_error` channel, so one
|
||||
stall unwinds the whole rank through one path rather than each waiter
|
||||
timing out separately — here both a consumer parked in the serve cache
|
||||
wait and the engine parked in its credit gate."""
|
||||
wait and the engine parked in its credit gate.
|
||||
"""
|
||||
server = server_factory(gather_lookahead=1, stall_timeout_s=0.3)
|
||||
server.begin_sync(1)
|
||||
_publish(server, GI_A, GROUP_A)
|
||||
@@ -796,7 +819,8 @@ class TestStallWatchdog:
|
||||
"""A consumer that is slow but signaling must never trip it: the stamp
|
||||
is global to the producer, so a steady trickle of signals holds the
|
||||
whole rank open. Nine sequential publishes with a 0.3s timeout each take
|
||||
longer in total than the timeout, and none may fire."""
|
||||
longer in total than the timeout, and none may fire.
|
||||
"""
|
||||
server = server_factory(gather_lookahead=1, stall_timeout_s=0.3)
|
||||
server.begin_sync(1)
|
||||
for gi in range(9):
|
||||
@@ -809,7 +833,8 @@ class TestStallWatchdog:
|
||||
|
||||
def test_begin_sync_resets_the_progress_stamp(self, server_factory):
|
||||
"""Syncs are minutes apart. Without the reset the first publish of sync N+1
|
||||
would measure its stall from somewhere inside sync N and fire immediately."""
|
||||
would measure its stall from somewhere inside sync N and fire immediately.
|
||||
"""
|
||||
server = server_factory(gather_lookahead=1, stall_timeout_s=0.3)
|
||||
server.begin_sync(1)
|
||||
_publish(server, GI_A, GROUP_A)
|
||||
@@ -831,7 +856,8 @@ class TestConcurrentPublishAndFree:
|
||||
self, server_factory
|
||||
):
|
||||
"""Every group published must end up freed and out of the cache, with
|
||||
end_sync returning cleanly — the property whose violation is a hang."""
|
||||
end_sync returning cleanly — the property whose violation is a hang.
|
||||
"""
|
||||
server = server_factory(gather_lookahead=2)
|
||||
server.begin_sync(1)
|
||||
groups = [(gi, (f"model.layers.{gi}.w",)) for gi in range(12)]
|
||||
@@ -866,7 +892,8 @@ class TestConcurrentPublishAndFree:
|
||||
):
|
||||
"""All live consumers signaling at once: the release must happen on the
|
||||
last signal and only once, or the engine drops IPC refs twice (or
|
||||
never)."""
|
||||
never).
|
||||
"""
|
||||
server = server_factory()
|
||||
server.begin_sync(8)
|
||||
_publish(server, GI_A, GROUP_A)
|
||||
@@ -893,7 +920,8 @@ class TestFakeServerAgreesWithTheRealOne:
|
||||
"""`test_sharded_rdt_trainer.py`'s `_FakeProducerServer` is a second,
|
||||
independent model of this protocol that the engine-side tests assert on. Pin
|
||||
the two against each other so the fake cannot silently drift from the
|
||||
semantics the real server enforces."""
|
||||
semantics the real server enforces.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _fake():
|
||||
@@ -924,7 +952,8 @@ class TestExportRing:
|
||||
"""The packed export path. Its one hard invariant is that the trainer packs
|
||||
a group exactly the way ``publish_group`` rebuilds it — a layout
|
||||
disagreement would serve the wrong bytes with nothing downstream to catch
|
||||
it."""
|
||||
it.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _engine():
|
||||
@@ -1016,7 +1045,8 @@ class TestExportRingSlotSafety:
|
||||
):
|
||||
"""Free the later group of each pair first and hold the earlier one live
|
||||
until the NEXT group is gathered -- the window a modular slot counter got
|
||||
wrong (at lookahead=1 there are 2 slots, so group 2 took group 0's)."""
|
||||
wrong (at lookahead=1 there are 2 slots, so group 2 took group 0's).
|
||||
"""
|
||||
server = server_factory(gather_lookahead=1)
|
||||
n_groups = 8
|
||||
engine = gather_engine(server, n_groups, lookahead=1)
|
||||
@@ -1140,7 +1170,8 @@ class TestSharedServeSlots:
|
||||
|
||||
def test_one_deployment_never_shares(self, sharing_server):
|
||||
"""With the width equal to the live count every group is a singleton, so
|
||||
each call packs for itself out of its own ring."""
|
||||
each call packs for itself out of its own ring.
|
||||
"""
|
||||
server = sharing_server(workers_per_replica=2)
|
||||
server.begin_sync(2, [0, 1])
|
||||
_publish(server, GI_A, GROUP_A)
|
||||
@@ -1220,7 +1251,8 @@ class TestSharedServeSlots:
|
||||
|
||||
def test_a_seqless_call_is_refused(self, sharing_server):
|
||||
"""The slot comes from ``seq``, so a caller that sends none cannot be
|
||||
served safely."""
|
||||
served safely.
|
||||
"""
|
||||
server = sharing_server(workers_per_replica=2)
|
||||
server.begin_sync(1, [0])
|
||||
_publish(server, GI_A, GROUP_A)
|
||||
@@ -1230,7 +1262,8 @@ class TestSharedServeSlots:
|
||||
|
||||
def test_a_failed_pack_reaches_every_sharer(self, sharing_server):
|
||||
"""Only the packer touches the buffer, so its failure has to be
|
||||
republished: a waiter must raise rather than read a half-written slot."""
|
||||
republished: a waiter must raise rather than read a half-written slot.
|
||||
"""
|
||||
server = sharing_server(workers_per_replica=2)
|
||||
server.begin_sync(4, [0, 1, 2, 3])
|
||||
_publish(server, GI_A, GROUP_A)
|
||||
@@ -1254,7 +1287,8 @@ class TestSharedServeSlots:
|
||||
|
||||
def test_a_degraded_sync_narrows_the_rendezvous(self, sharing_server):
|
||||
"""A dead deployment must not be waited for: a survivor left alone in
|
||||
its group packs for itself instead of blocking on the corpse."""
|
||||
its group packs for itself instead of blocking on the corpse.
|
||||
"""
|
||||
server = sharing_server(workers_per_replica=2, stall_timeout_s=30.0)
|
||||
server.begin_sync(3, [0, 1, 3]) # consumer 2 died
|
||||
_publish(server, GI_A, GROUP_A)
|
||||
@@ -1268,7 +1302,8 @@ class TestSharedServeSlots:
|
||||
|
||||
def test_a_sharer_that_never_arrives_trips_the_watchdog(self, sharing_server):
|
||||
"""A consumer that dies INSIDE the sync window has no detector, exactly
|
||||
as for the free barrier; the rendezvous waits on the same channel."""
|
||||
as for the free barrier; the rendezvous waits on the same channel.
|
||||
"""
|
||||
server = sharing_server(workers_per_replica=2, stall_timeout_s=0.3)
|
||||
server.begin_sync(4, [0, 1, 2, 3])
|
||||
_publish(server, GI_A, GROUP_A)
|
||||
@@ -1280,7 +1315,8 @@ class TestSharedServeSlots:
|
||||
|
||||
class TestSharedReservation:
|
||||
"""``reserve_serve_buffer`` under sharing: one ring per GROUP, and the
|
||||
init-time check that the group's members pull the same chunks."""
|
||||
init-time check that the group's members pull the same chunks.
|
||||
"""
|
||||
|
||||
def test_matching_plans_reserve_one_ring_between_them(self, sharing_server):
|
||||
server = sharing_server(workers_per_replica=2)
|
||||
@@ -1302,7 +1338,8 @@ class TestSharedReservation:
|
||||
def test_mismatched_plans_fail_at_reservation(self, sharing_server):
|
||||
"""Sharing rests on the sharers pulling the same chunks. Checked here,
|
||||
where it names both consumers, rather than at serve time, where it is a
|
||||
rendezvous nobody completes."""
|
||||
rendezvous nobody completes.
|
||||
"""
|
||||
server = sharing_server(workers_per_replica=2)
|
||||
server.reserve_serve_buffer(0, 1024, "plan-a")
|
||||
|
||||
@@ -1311,7 +1348,8 @@ class TestSharedReservation:
|
||||
|
||||
def test_different_groups_may_differ(self, sharing_server):
|
||||
"""Only the members of ONE group have to agree; worker 0 and worker 1 of
|
||||
the same deployment hold different slices and never share a slot."""
|
||||
the same deployment hold different slices and never share a slot.
|
||||
"""
|
||||
server = sharing_server(workers_per_replica=2)
|
||||
server.reserve_serve_buffer(0, 1024, "plan-a")
|
||||
server.reserve_serve_buffer(1, 1024, "plan-b")
|
||||
@@ -1320,7 +1358,8 @@ class TestSharedReservation:
|
||||
|
||||
def test_a_missing_digest_skips_the_check(self, sharing_server):
|
||||
"""The digest is optional on the wire, so an engine that does not send
|
||||
one still reserves."""
|
||||
one still reserves.
|
||||
"""
|
||||
server = sharing_server(workers_per_replica=2)
|
||||
server.reserve_serve_buffer(0, 1024, None)
|
||||
server.reserve_serve_buffer(2, 1024, None)
|
||||
|
||||
@@ -34,7 +34,8 @@ from vllm.distributed.weight_transfer.sharded_rdt_trainer import (
|
||||
|
||||
class _ListSource(WeightSource):
|
||||
"""A WeightSource over an explicit ordered (name, cpu-tensor) list, so the
|
||||
sharded-RDT group/order logic can be tested without a real model."""
|
||||
sharded-RDT group/order logic can be tested without a real model.
|
||||
"""
|
||||
|
||||
def __init__(self, pairs):
|
||||
self._pairs = list(pairs)
|
||||
@@ -54,7 +55,8 @@ class _FakeProducerServer:
|
||||
per-group barrier: publish/free are keyed by GROUP INDEX, signals count
|
||||
to the ``begin_sync`` live total, publish_group returns nothing, and freed
|
||||
groups flow back only through wait_freed / end_sync (see
|
||||
test_sharded_rdt_producer.TestFakeServerAgreesWithTheRealOne)."""
|
||||
test_sharded_rdt_producer.TestFakeServerAgreesWithTheRealOne).
|
||||
"""
|
||||
|
||||
def __init__(self, auto_free=True):
|
||||
self.order: list[str] = []
|
||||
@@ -98,7 +100,8 @@ class _FakeProducerServer:
|
||||
must already have a banked credit when the gate asks (auto_free, or a
|
||||
test's own free_group) — anything else is the deadlock the real
|
||||
watchdog would kill, so fail loudly. Not appended to ``order``: it is
|
||||
pacing, not a lifecycle milestone."""
|
||||
pacing, not a lifecycle milestone.
|
||||
"""
|
||||
assert self._pending_freed, (
|
||||
"wait_freed with nothing freed: the gather loop would deadlock"
|
||||
)
|
||||
@@ -121,7 +124,8 @@ def _rdt_engine_with_fake_server(
|
||||
):
|
||||
"""Build a ShardedRDTTrainerWeightTransferEngine wired to an in-process fake
|
||||
server (no Ray, no CUDA IPC): bypass trainer_init's spawn, set the
|
||||
group-major metadata, and route _rpc to the fake."""
|
||||
group-major metadata, and route _rpc to the fake.
|
||||
"""
|
||||
import vllm.distributed.weight_transfer.sharded_rdt_trainer as mod
|
||||
|
||||
# reduce_tensor needs CUDA; the fake server never rebuilds, so stub it.
|
||||
@@ -321,7 +325,8 @@ def test_sharded_rdt_send_weights_surfaces_update_error(monkeypatch):
|
||||
|
||||
class _OwnedSource(_ListSource):
|
||||
"""A source holding only some groups' names, like a pipeline-parallel rank.
|
||||
Takes group indices for convenience and declares the names inside them."""
|
||||
Takes group indices for convenience and declares the names inside them.
|
||||
"""
|
||||
|
||||
def __init__(self, pairs, owned_group_idx):
|
||||
super().__init__(pairs)
|
||||
@@ -387,7 +392,8 @@ def test_sharded_rdt_owned_group_order_mismatch_raises(monkeypatch):
|
||||
)
|
||||
def test_sharded_rdt_begin_sync_carries_the_live_count(monkeypatch):
|
||||
"""The free barrier's target is one integer per sync: the live consumer
|
||||
count, defaulting to the whole provisioned fleet."""
|
||||
count, defaulting to the whole provisioned fleet.
|
||||
"""
|
||||
server = _FakeProducerServer(auto_free=True)
|
||||
engine = _rdt_engine_with_fake_server(
|
||||
_rdt_source_two_layers(),
|
||||
@@ -405,7 +411,8 @@ def test_sharded_rdt_begin_sync_carries_the_live_count(monkeypatch):
|
||||
class TestLiveCountPlumbing:
|
||||
"""``send_weights(live_consumer_ids)`` -> the barrier target. The
|
||||
provisioned geometry is frozen; a degraded sync only lowers the target and
|
||||
narrows each slot-sharing group's rendezvous."""
|
||||
narrows each slot-sharing group's rendezvous.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _engine(num_consumers, world=2, rank=0):
|
||||
@@ -438,7 +445,8 @@ class TestLiveCountPlumbing:
|
||||
|
||||
def test_which_consumers_died_does_not_matter_only_how_many(self):
|
||||
"""The whole point of the barrier: no routed per-producer targets, so
|
||||
the identity of the dead consumer is irrelevant to the producers."""
|
||||
the identity of the dead consumer is irrelevant to the producers.
|
||||
"""
|
||||
counts = []
|
||||
for live in ([0, 1, 2, 3], [4, 5, 6, 7], [0, 2, 4, 6]):
|
||||
engine, got = self._engine(num_consumers=8)
|
||||
@@ -449,7 +457,8 @@ class TestLiveCountPlumbing:
|
||||
def test_the_live_ids_travel_with_the_count(self):
|
||||
"""The count sizes the free barrier, the ids size the slot-sharing
|
||||
rendezvous, so a producer that shares slots can tell WHICH consumers it
|
||||
is still waiting for. They must describe the same set."""
|
||||
is still waiting for. They must describe the same set.
|
||||
"""
|
||||
engine, got = self._engine(num_consumers=8)
|
||||
engine.send_weights([6, 0, 2, 2])
|
||||
((count, ids),) = got
|
||||
@@ -517,7 +526,8 @@ def test_sharded_rdt_rejects_a_rank_holding_nothing(monkeypatch):
|
||||
|
||||
def test_sharded_rdt_rejects_a_name_no_rank_holds(monkeypatch):
|
||||
"""Every name must be held somewhere or it can never be served — caught
|
||||
when the holdings are transposed, naming the orphan."""
|
||||
when the holdings are transposed, naming the orphan.
|
||||
"""
|
||||
with pytest.raises(ValueError, match="no trainer rank holds"):
|
||||
_rdt_engine_with_fake_server(
|
||||
_OwnedSource(_rdt_source_two_layers()._pairs, [0, 1]),
|
||||
@@ -531,7 +541,8 @@ def test_sharded_rdt_rejects_a_name_no_rank_holds(monkeypatch):
|
||||
|
||||
def test_sharded_rdt_rejects_metadata_disagreement_across_ranks(monkeypatch):
|
||||
"""Only the sender's metadata reaches the consumers, so a rank describing
|
||||
just its own share must fail loudly rather than silently drop the rest."""
|
||||
just its own share must fail loudly rather than silently drop the rest.
|
||||
"""
|
||||
engine = _rdt_engine_with_fake_server(
|
||||
_OwnedSource(_rdt_source_two_layers()._pairs, [1, 2]),
|
||||
is_sender=False,
|
||||
@@ -554,7 +565,8 @@ def _serve_ring_server(src_name, src):
|
||||
"""A producer server with one cached tensor and a pre-seeded serve ring, so a
|
||||
pull needs no Ray and no NIXL registration. Returns (server, serve) where
|
||||
``serve(chain)`` packs one spec into the SAME ring slot every time — which is
|
||||
what puts the destination-view cache, and only it, under test."""
|
||||
what puts the destination-view cache, and only it, under test.
|
||||
"""
|
||||
from vllm.distributed.weight_transfer.sharded_rdt_trainer import (
|
||||
_RDTProducerServer,
|
||||
)
|
||||
|
||||
@@ -43,6 +43,7 @@ def distributed_run(fn, world_size, timeout=60):
|
||||
fn: Function to run in each process
|
||||
world_size: Number of processes to spawn
|
||||
timeout: Maximum time in seconds to wait for processes (default: 60)
|
||||
|
||||
"""
|
||||
number_of_processes = world_size
|
||||
processes = []
|
||||
@@ -418,7 +419,8 @@ def test_tensor_broadcast():
|
||||
|
||||
def _dumps_oob(obj) -> tuple[bytes, list]:
|
||||
"""Pickle `obj` the same way `MessageQueue.enqueue` does: tensor
|
||||
dispatch table + out-of-band buffers >= 1MiB."""
|
||||
dispatch table + out-of-band buffers >= 1MiB.
|
||||
"""
|
||||
buffers = []
|
||||
|
||||
def callback(buf: pickle.PickleBuffer) -> bool:
|
||||
@@ -489,7 +491,8 @@ def test_tensor_pickle_roundtrip(case: str):
|
||||
@pytest.mark.parametrize("case", ["cuda", "requires_grad", "conj"])
|
||||
def test_reduce_tensor_fallback(case: str):
|
||||
"""Tensors the zero-copy reducer can't safely alias must fall back to
|
||||
torch's default reduction."""
|
||||
torch's default reduction.
|
||||
"""
|
||||
if case == "cuda":
|
||||
if not torch.cuda.is_available():
|
||||
pytest.skip("requires CUDA")
|
||||
@@ -634,11 +637,9 @@ def test_acquire_read_releases_slot_when_reader_raises():
|
||||
|
||||
|
||||
def test_warning_logs(caplog_vllm):
|
||||
"""
|
||||
Test that warning logs are emitted at VLLM_RINGBUFFER_WARNING_INTERVAL intervals
|
||||
"""Test that warning logs are emitted at VLLM_RINGBUFFER_WARNING_INTERVAL intervals
|
||||
when indefinite=False, and are not emitted when indefinite=True.
|
||||
"""
|
||||
|
||||
# Patch the warning log interval to every 1 ms during reads
|
||||
with mock.patch(
|
||||
"vllm.distributed.device_communicators.shm_broadcast.VLLM_RINGBUFFER_WARNING_INTERVAL",
|
||||
@@ -754,7 +755,8 @@ def test_remote_subscribe_addr_unique_concurrent_writers(
|
||||
Pre-fix, the writer probed a port with get_open_port() and bound it
|
||||
afterwards; pinning the probe to one free port makes every writer
|
||||
bind the same port and fail deterministically on that code path,
|
||||
while the late-binding implementation never consults the probe."""
|
||||
while the late-binding implementation never consults the probe.
|
||||
"""
|
||||
from vllm.distributed.device_communicators import shm_broadcast
|
||||
|
||||
colliding_port = get_open_port()
|
||||
|
||||
@@ -12,20 +12,20 @@ from vllm.distributed.device_communicators.shm_object_storage import (
|
||||
|
||||
|
||||
class TestSingleWriterShmRingBuffer(unittest.TestCase):
|
||||
"""Test suite for the ring buffer implementation"""
|
||||
"""Test suite for the ring buffer implementation."""
|
||||
|
||||
def setUp(self):
|
||||
"""Set up test fixtures"""
|
||||
"""Set up test fixtures."""
|
||||
self.buffer_size = 4096
|
||||
self.ring_buffer = None
|
||||
|
||||
def tearDown(self):
|
||||
"""Clean up after tests"""
|
||||
"""Clean up after tests."""
|
||||
if self.ring_buffer:
|
||||
self.ring_buffer.close()
|
||||
|
||||
def test_buffer_opening(self):
|
||||
"""Test opening an existing buffer"""
|
||||
"""Test opening an existing buffer."""
|
||||
# First create a buffer
|
||||
self.ring_buffer = SingleWriterShmRingBuffer(
|
||||
data_buffer_size=self.buffer_size, create=True
|
||||
@@ -39,7 +39,7 @@ class TestSingleWriterShmRingBuffer(unittest.TestCase):
|
||||
)
|
||||
|
||||
def test_buffer_access(self):
|
||||
"""Test accessing allocated buffers"""
|
||||
"""Test accessing allocated buffers."""
|
||||
self.ring_buffer = SingleWriterShmRingBuffer(
|
||||
data_buffer_size=self.buffer_size, create=True
|
||||
)
|
||||
@@ -61,7 +61,7 @@ class TestSingleWriterShmRingBuffer(unittest.TestCase):
|
||||
self.assertEqual(read_id, monotonic_id)
|
||||
|
||||
def test_memory_error_on_full_buffer(self):
|
||||
"""Test that MemoryError is raised when buffer is full"""
|
||||
"""Test that MemoryError is raised when buffer is full."""
|
||||
small_buffer_size = 200
|
||||
self.ring_buffer = SingleWriterShmRingBuffer(
|
||||
data_buffer_size=small_buffer_size, create=True
|
||||
@@ -76,7 +76,7 @@ class TestSingleWriterShmRingBuffer(unittest.TestCase):
|
||||
self.ring_buffer.allocate_buf(1) # Would exceed buffer capacity
|
||||
|
||||
def test_allocation_and_free(self):
|
||||
"""Test allocation and freeing of buffers"""
|
||||
"""Test allocation and freeing of buffers."""
|
||||
small_buffer_size = 200
|
||||
self.ring_buffer = SingleWriterShmRingBuffer(
|
||||
data_buffer_size=small_buffer_size, create=True
|
||||
@@ -96,7 +96,7 @@ class TestSingleWriterShmRingBuffer(unittest.TestCase):
|
||||
self.assertEqual(freed_ids[0], i)
|
||||
|
||||
def test_clear_buffer(self):
|
||||
"""Test clearing the buffer"""
|
||||
"""Test clearing the buffer."""
|
||||
self.ring_buffer = SingleWriterShmRingBuffer(
|
||||
data_buffer_size=self.buffer_size, create=True
|
||||
)
|
||||
@@ -180,7 +180,7 @@ class TestSingleWriterShmRingBuffer(unittest.TestCase):
|
||||
|
||||
|
||||
def main():
|
||||
"""Main function demonstrating usage and running tests"""
|
||||
"""Main function demonstrating usage and running tests."""
|
||||
print("=== SingleWriterShmRingBuffer Test Suite ===\n")
|
||||
|
||||
# Run unit tests
|
||||
|
||||
@@ -31,7 +31,8 @@ class _CUDADeviceCountStatelessTestActor:
|
||||
|
||||
def test_cuda_device_count_stateless():
|
||||
"""Test that cuda_device_count_stateless changes return value if
|
||||
CUDA_VISIBLE_DEVICES is changed."""
|
||||
CUDA_VISIBLE_DEVICES is changed.
|
||||
"""
|
||||
if current_platform.is_rocm():
|
||||
pytest.skip("Skip for ROCm because Ray uses HIP_VISIBLE_DEVICES.")
|
||||
actor = _CUDADeviceCountStatelessTestActor.options( # type: ignore
|
||||
|
||||
@@ -1166,7 +1166,8 @@ class RecordingClient:
|
||||
|
||||
def _module_with(*pairs):
|
||||
"""A tiny nn.Module exposing the given (name, tensor) pairs as parameters,
|
||||
so trainer tests can build a ModuleSource without a real model."""
|
||||
so trainer tests can build a ModuleSource without a real model.
|
||||
"""
|
||||
module = torch.nn.Module()
|
||||
for name, tensor in pairs:
|
||||
module.register_parameter(name, torch.nn.Parameter(tensor, requires_grad=False))
|
||||
@@ -1287,7 +1288,8 @@ class TestModuleSource:
|
||||
|
||||
def test_metadata_agrees_with_iteration(self):
|
||||
"""The two channels must line up element-for-element: engines declare
|
||||
the round from `metadata()` and then send what iteration yields."""
|
||||
the round from `metadata()` and then send what iteration yields.
|
||||
"""
|
||||
source = ModuleSource(
|
||||
_module_with(("w", torch.zeros(2, 3)), ("b", torch.zeros(3)))
|
||||
)
|
||||
@@ -1301,11 +1303,13 @@ class TestModuleSource:
|
||||
class TestWeightSourceGroupContract:
|
||||
"""`groups()` / `iter_groups()` on the WeightSource ABC. Groups are what
|
||||
backends gather and free by, so the default must agree with
|
||||
`layerwise_groups` over `metadata()`, restricted to what this rank holds."""
|
||||
`layerwise_groups` over `metadata()`, restricted to what this rank holds.
|
||||
"""
|
||||
|
||||
class _Source(WeightSource):
|
||||
"""Minimal source over an ordered (name, tensor) list, optionally holding
|
||||
only some names (in which case it iterates only their groups)."""
|
||||
only some names (in which case it iterates only their groups).
|
||||
"""
|
||||
|
||||
def __init__(self, names, held=None, reverse=False):
|
||||
self._pairs = [(n, torch.full((2,), float(i))) for i, n in enumerate(names)]
|
||||
@@ -1360,7 +1364,8 @@ class TestWeightSourceGroupContract:
|
||||
def test_groups_order_follows_metadata_not_the_declaration(self):
|
||||
"""The declaration is a SET of names; the groups it selects still come
|
||||
out in metadata order, so each pairs with the right ``iter_groups()``
|
||||
batch however the source listed them."""
|
||||
batch however the source listed them.
|
||||
"""
|
||||
names = ["embed.w", "model.layers.0.a", "model.layers.1.a", "norm.w"]
|
||||
source = self._source(
|
||||
names, held=["model.layers.1.a", "model.layers.0.a", "model.layers.1.a"]
|
||||
@@ -1394,14 +1399,16 @@ class TestWeightSourceGroupContract:
|
||||
|
||||
def test_out_of_order_iteration_raises(self):
|
||||
"""Materializing is usually a collective, so a rank that iterates out of
|
||||
order deadlocks its peers -- fail loudly instead."""
|
||||
order deadlocks its peers -- fail loudly instead.
|
||||
"""
|
||||
source = self._source(["model.layers.0.a", "model.layers.0.b"], reverse=True)
|
||||
with pytest.raises(RuntimeError, match="iteration order must match"):
|
||||
list(source.iter_groups())
|
||||
|
||||
def test_a_source_may_override_iter_groups(self):
|
||||
"""The extension point: materialize a whole group in one step instead of
|
||||
one generator resume per tensor."""
|
||||
one generator resume per tensor.
|
||||
"""
|
||||
calls = []
|
||||
|
||||
class _Batched(TestWeightSourceGroupContract._Source):
|
||||
@@ -1422,7 +1429,8 @@ class TestDeferredProcessingContract:
|
||||
caller that takes over the update tail (running its own
|
||||
`finalize_layerwise_reload` instead of going through `finish_weight_update`)
|
||||
reads the flag and calls the method. Both must be answerable on any engine, or
|
||||
that caller ends up reaching through a getattr."""
|
||||
that caller ends up reaching through a getattr.
|
||||
"""
|
||||
|
||||
def _engines(self):
|
||||
return {
|
||||
@@ -1436,7 +1444,8 @@ class TestDeferredProcessingContract:
|
||||
|
||||
def test_every_engine_can_be_drained(self):
|
||||
"""The default is a no-op, so a caller never has to check whether the
|
||||
method exists before calling it."""
|
||||
method exists before calling it.
|
||||
"""
|
||||
for name, cls in self._engines().items():
|
||||
assert callable(cls.drain_pending), name
|
||||
|
||||
@@ -1530,7 +1539,8 @@ class TestTrainerEngineBase:
|
||||
)
|
||||
def test_ipc_trainer_send_weights_drives_client_in_order():
|
||||
"""send_weights issues start -> update -> finish and ships per-round metadata;
|
||||
the packed wire param rides the init info, not the per-round update_info."""
|
||||
the packed wire param rides the init info, not the per-round update_info.
|
||||
"""
|
||||
client = RecordingClient()
|
||||
engine = IPCTrainerWeightTransferEngine(
|
||||
client=client,
|
||||
@@ -1549,7 +1559,8 @@ def test_ipc_trainer_send_weights_drives_client_in_order():
|
||||
|
||||
def test_ipc_trainer_init_ships_packed_to_worker():
|
||||
"""trainer_init drives the inference-side init handshake and propagates the
|
||||
must-agree `packed` flag to the worker."""
|
||||
must-agree `packed` flag to the worker.
|
||||
"""
|
||||
if torch.accelerator.device_count() < 1:
|
||||
pytest.skip("Need at least 1 GPU (CUDA IPC handles).")
|
||||
|
||||
@@ -1570,7 +1581,8 @@ def test_ipc_trainer_init_ships_packed_to_worker():
|
||||
def test_nccl_trainer_init_ships_worker_init_info(monkeypatch):
|
||||
"""The sender's trainer_init drives the inference-side init handshake with
|
||||
the worker-shaped init info (rank_offset=1) while opening its own endpoint,
|
||||
and propagates the must-agree wire params to the worker."""
|
||||
and propagates the must-agree wire params to the worker.
|
||||
"""
|
||||
import vllm.distributed.weight_transfer.nccl_engine as nccl_engine_mod
|
||||
|
||||
# Bypass the real NCCL rendezvous.
|
||||
@@ -1611,7 +1623,8 @@ def test_nccl_trainer_init_ships_worker_init_info(monkeypatch):
|
||||
def test_nccl_worker_learns_wire_params_from_init_handshake(monkeypatch):
|
||||
"""The worker engine reads packed + buffer geometry from the
|
||||
trainer-supplied init info at the handshake, not from the config or the
|
||||
per-round update info."""
|
||||
per-round update info.
|
||||
"""
|
||||
import vllm.distributed.weight_transfer.nccl_engine as nccl_engine_mod
|
||||
|
||||
monkeypatch.setattr(
|
||||
@@ -1644,7 +1657,8 @@ def test_nccl_worker_learns_wire_params_from_init_handshake(monkeypatch):
|
||||
|
||||
def test_nccl_trainer_init_non_sender_skips_rendezvous_and_client():
|
||||
"""Non-sender trainer ranks build an engine without opening an endpoint or
|
||||
touching the client; they only join the collectives in send_weights."""
|
||||
touching the client; they only join the collectives in send_weights.
|
||||
"""
|
||||
client = RecordingClient()
|
||||
engine = WeightTransferTrainerFactory.trainer_init(
|
||||
init_info=NCCLTrainerInitInfo(
|
||||
@@ -1674,7 +1688,8 @@ def test_nccl_trainer_init_non_sender_skips_rendezvous_and_client():
|
||||
def test_nccl_trainer_send_weights_drives_client_in_order():
|
||||
"""send_weights issues start -> update -> finish and ships per-round
|
||||
metadata; the packed wire params ride the init handshake, not the
|
||||
per-round update_info."""
|
||||
per-round update_info.
|
||||
"""
|
||||
client = RecordingClient()
|
||||
engine = NCCLTrainerWeightTransferEngine(
|
||||
client=client,
|
||||
@@ -1695,7 +1710,8 @@ def test_nccl_trainer_send_weights_drives_client_in_order():
|
||||
|
||||
class _ScriptedSource(WeightSource):
|
||||
"""Declares `meta` but yields whatever `pairs` says — used to drive the
|
||||
metadata/iteration agreement checks."""
|
||||
metadata/iteration agreement checks.
|
||||
"""
|
||||
|
||||
def __init__(self, meta, pairs):
|
||||
self._meta = meta
|
||||
@@ -1731,7 +1747,8 @@ def test_nccl_trainer_init_requires_source():
|
||||
|
||||
def test_nccl_trainer_send_weights_rejects_reordered_source(monkeypatch):
|
||||
"""The worker sizes its buffers (and cuts packed chunks) from metadata(), so
|
||||
iteration disagreeing with it must raise rather than corrupt the stream."""
|
||||
iteration disagreeing with it must raise rather than corrupt the stream.
|
||||
"""
|
||||
meta = [
|
||||
ParamMeta("w", torch.float32, (4,)),
|
||||
ParamMeta("b", torch.float32, (2,)),
|
||||
@@ -1745,7 +1762,8 @@ def test_nccl_trainer_send_weights_rejects_reordered_source(monkeypatch):
|
||||
|
||||
def test_nccl_trainer_send_weights_rejects_dtype_disagreement(monkeypatch):
|
||||
"""A source that declares one wire dtype and materializes another would make
|
||||
the two sides disagree on every byte offset."""
|
||||
the two sides disagree on every byte offset.
|
||||
"""
|
||||
meta = [ParamMeta("w", torch.float32, (4,))]
|
||||
engine = _mock_group_engine(
|
||||
_ScriptedSource(meta, [("w", torch.zeros(4, dtype=torch.bfloat16))]),
|
||||
@@ -1772,7 +1790,8 @@ def test_nccl_trainer_send_weights_rejects_truncated_source(monkeypatch):
|
||||
|
||||
def test_nccl_trainer_send_weights_broadcasts_contiguous(monkeypatch):
|
||||
"""NCCL sends numel elements from data_ptr(), so a non-contiguous view must
|
||||
be linearized first or the worker receives unrelated memory."""
|
||||
be linearized first or the worker receives unrelated memory.
|
||||
"""
|
||||
base = torch.arange(6, dtype=torch.float32).reshape(2, 3)
|
||||
view = base.t() # non-contiguous
|
||||
meta = [ParamMeta("w", torch.float32, tuple(view.shape))]
|
||||
@@ -1842,7 +1861,8 @@ def _sparse_patch(device: str = "cpu") -> SparseWeightPatch:
|
||||
def test_sparse_nccl_trainer_init_ships_worker_init_info(monkeypatch):
|
||||
"""The sender's trainer_init drives the init handshake with the
|
||||
worker-shaped init info; sparse ships no packed wire params, so the worker
|
||||
keeps its unpacked defaults. Sparse takes no `source`."""
|
||||
keeps its unpacked defaults. Sparse takes no `source`.
|
||||
"""
|
||||
import vllm.distributed.weight_transfer.sparse_nccl_engine as sparse_mod
|
||||
|
||||
monkeypatch.setattr(sparse_mod, "open_trainer_endpoint", lambda info: MagicMock())
|
||||
@@ -1895,7 +1915,8 @@ def test_sparse_nccl_trainer_send_weights_drives_client_in_order(monkeypatch):
|
||||
|
||||
def test_sparse_nccl_trainer_send_weights_empty_round_is_noop():
|
||||
"""A round with no patches must not touch the client (an empty sparse
|
||||
update info is invalid by construction)."""
|
||||
update info is invalid by construction).
|
||||
"""
|
||||
client = RecordingClient()
|
||||
engine = SparseNCCLTrainerWeightTransferEngine(client=client)
|
||||
engine.model_update_group = MagicMock()
|
||||
@@ -1931,7 +1952,8 @@ def test_sparse_nccl_trainer_rejects_source():
|
||||
def test_sparse_nccl_trainer_validates_patch_before_any_rpc():
|
||||
"""Malformed patches must fail on the trainer, before start_weight_update:
|
||||
the worker's own checks only run once the broadcasts are already in flight,
|
||||
where a size mismatch wedges both sides instead of raising."""
|
||||
where a size mismatch wedges both sides instead of raising.
|
||||
"""
|
||||
client = RecordingClient()
|
||||
engine = SparseNCCLTrainerWeightTransferEngine(client=client)
|
||||
engine.model_update_group = MagicMock()
|
||||
|
||||
@@ -403,7 +403,8 @@ def trainer_broadcast_tensor_torch_free(
|
||||
def _run_uid_transfer(trainer_task) -> None:
|
||||
"""Mint a unique id, run the worker and the given trainer peer concurrently,
|
||||
and assert the tensor round-trips. Both ranks must enter init together --
|
||||
there is no store barrier on the UID path."""
|
||||
there is no store barrier on the UID path.
|
||||
"""
|
||||
from vllm.distributed.device_communicators.pynccl_wrapper import NCCLLibrary
|
||||
|
||||
_init_ray_for_weight_transfer()
|
||||
@@ -450,7 +451,8 @@ def _run_uid_transfer(trainer_task) -> None:
|
||||
)
|
||||
def test_nccl_weight_transfer_between_processes_uid():
|
||||
"""Weight transfer over a pre-shared ncclUniqueId (no TCPStore), with both
|
||||
ranks on vLLM's ``PyNcclCommunicator``."""
|
||||
ranks on vLLM's ``PyNcclCommunicator``.
|
||||
"""
|
||||
_run_uid_transfer(trainer_broadcast_tensor_uid)
|
||||
|
||||
|
||||
@@ -460,6 +462,7 @@ def test_nccl_weight_transfer_between_processes_uid():
|
||||
)
|
||||
def test_nccl_weight_transfer_torch_free_trainer():
|
||||
"""UID weight transfer against a foreign peer that uses no torch and no vLLM
|
||||
communicator, only a cupy buffer and raw ``ncclBroadcast``."""
|
||||
communicator, only a cupy buffer and raw ``ncclBroadcast``.
|
||||
"""
|
||||
pytest.importorskip("cupy")
|
||||
_run_uid_transfer(trainer_broadcast_tensor_torch_free)
|
||||
|
||||
@@ -394,8 +394,7 @@ def test_media_io_kwargs_parser(arg, expected):
|
||||
],
|
||||
)
|
||||
def test_optimization_level(args, expected):
|
||||
"""
|
||||
Test space-separated optimization levels (-O 1, -O 2, -O 3) map to
|
||||
"""Test space-separated optimization levels (-O 1, -O 2, -O 3) map to
|
||||
optimization_level.
|
||||
"""
|
||||
parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
|
||||
@@ -414,9 +413,7 @@ def test_optimization_level(args, expected):
|
||||
],
|
||||
)
|
||||
def test_mode_parser(args, expected):
|
||||
"""
|
||||
Test compilation config modes (-cc.mode=int) map to compilation_config.
|
||||
"""
|
||||
"""Test compilation config modes (-cc.mode=int) map to compilation_config."""
|
||||
parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
|
||||
parsed_args = parser.parse_args(args)
|
||||
assert parsed_args.compilation_config.mode == expected
|
||||
@@ -573,7 +570,8 @@ def test_attention_config():
|
||||
def test_multi_node_world_size_includes_pcp(monkeypatch):
|
||||
"""PCP expands the process world size, so the --nnodes divisibility check
|
||||
must include it. Without this, TP=1/PCP=2 over 2 nodes computes a world
|
||||
size of 1 and the launch is rejected before the engine starts."""
|
||||
size of 1 and the launch is rejected before the engine starts.
|
||||
"""
|
||||
import vllm.config.vllm
|
||||
|
||||
# PCP requires the V2 model runner, which is gated on Triton.
|
||||
@@ -799,7 +797,8 @@ def test_expand_json_human_readable_numbers(input_json, expected_json):
|
||||
)
|
||||
def test_cloud_storage_uri_skips_get_model_path(uri, monkeypatch):
|
||||
"""Cloud storage URIs should not be passed to get_model_path()
|
||||
when HF_HUB_OFFLINE=1, as they are not valid HF repo IDs."""
|
||||
when HF_HUB_OFFLINE=1, as they are not valid HF repo IDs.
|
||||
"""
|
||||
import huggingface_hub
|
||||
|
||||
monkeypatch.setattr(huggingface_hub.constants, "HF_HUB_OFFLINE", True)
|
||||
@@ -811,7 +810,8 @@ def test_cloud_storage_uri_skips_get_model_path(uri, monkeypatch):
|
||||
|
||||
def test_cloud_storage_tokenizer_skips_get_model_path(monkeypatch):
|
||||
"""Cloud storage tokenizer URI should not be passed to
|
||||
get_model_path() when HF_HUB_OFFLINE=1."""
|
||||
get_model_path() when HF_HUB_OFFLINE=1.
|
||||
"""
|
||||
import huggingface_hub
|
||||
|
||||
monkeypatch.setattr(huggingface_hub.constants, "HF_HUB_OFFLINE", True)
|
||||
@@ -973,7 +973,8 @@ class TestDpDeviceIdSharding:
|
||||
def test_dp_supervisor_device_ids_stay_env_relative(self):
|
||||
"""Regression test: the DP supervisor must pass env-relative indices,
|
||||
not physical IDs, because each child re-resolves --device-ids
|
||||
against its inherited device-control env var."""
|
||||
against its inherited device-control env var.
|
||||
"""
|
||||
import argparse
|
||||
|
||||
from vllm.entrypoints.launchers.dp_supervisor import _build_device_ids
|
||||
@@ -1000,7 +1001,8 @@ class TestDpDeviceIdSharding:
|
||||
|
||||
def test_dp_rank_shards_user_assigned_gpu_ids(self):
|
||||
"""get_physical_gpu_ids_for_local_dp_rank slices the user-provided
|
||||
--device-ids list instead of recomputing from the env var."""
|
||||
--device-ids list instead of recomputing from the env var.
|
||||
"""
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.engine.utils import get_physical_gpu_ids_for_local_dp_rank
|
||||
|
||||
|
||||
@@ -241,7 +241,8 @@ class TestToolResultContent:
|
||||
self, tool_result_content
|
||||
) -> AnthropicMessagesRequest:
|
||||
"""Build a request with assistant tool_use followed by user
|
||||
tool_result."""
|
||||
tool_result.
|
||||
"""
|
||||
return _make_request(
|
||||
[
|
||||
{
|
||||
@@ -326,7 +327,8 @@ class TestToolResultContent:
|
||||
|
||||
def test_tool_result_with_text_and_image(self):
|
||||
"""Mixed text+image tool_result: text in tool msg, image in user
|
||||
msg."""
|
||||
msg.
|
||||
"""
|
||||
request = self._make_tool_result_request(
|
||||
[
|
||||
{"type": "text", "text": "Here is the screenshot"},
|
||||
@@ -423,7 +425,8 @@ class TestToolResultContent:
|
||||
class TestAttributionHeaderStripping:
|
||||
def test_billing_header_stripped_from_system(self):
|
||||
"""Claude Code's x-anthropic-billing-header block should be
|
||||
stripped to preserve prefix caching."""
|
||||
stripped to preserve prefix caching.
|
||||
"""
|
||||
request = _make_request(
|
||||
[{"role": "user", "content": "Hello"}],
|
||||
system=[
|
||||
@@ -480,7 +483,7 @@ class TestThinkingBlockConversion:
|
||||
"""
|
||||
|
||||
def test_thinking_plus_text_in_assistant_message(self):
|
||||
"""thinking + text → reasoning field + plain-string content."""
|
||||
"""Thinking + text → reasoning field + plain-string content."""
|
||||
request = _make_request(
|
||||
[
|
||||
{"role": "user", "content": "Write me some code."},
|
||||
@@ -542,7 +545,7 @@ class TestThinkingBlockConversion:
|
||||
assert asst.get("content") is None
|
||||
|
||||
def test_thinking_plus_tool_use_in_assistant_message(self):
|
||||
"""thinking + tool_use: reasoning field set, tool_calls populated."""
|
||||
"""Thinking + tool_use: reasoning field set, tool_calls populated."""
|
||||
request = _make_request(
|
||||
[
|
||||
{"role": "user", "content": "What is 2+2?"},
|
||||
@@ -1147,7 +1150,8 @@ class TestMessageStreamConverterToolUseContentBuffering:
|
||||
async def test_buffered_content_flushed_on_done_without_usage_chunk(self):
|
||||
"""Content buffered during tool_use must be emitted even if the
|
||||
stream jumps straight from finish_reason to [DONE], skipping the
|
||||
empty-choices usage chunk."""
|
||||
empty-choices usage chunk.
|
||||
"""
|
||||
|
||||
async def sse_input():
|
||||
yield _make_stream_chunk(
|
||||
@@ -1251,7 +1255,8 @@ class TestStreamingCacheUsageSemantics:
|
||||
async def test_streaming_cache_fields_absent_then_populated(self):
|
||||
"""First chunk lacks prompt_tokens_details (vLLM contract);
|
||||
message_start omits cache fields. The final chunk carries
|
||||
prompt_tokens_details, so message_delta carries resolved values."""
|
||||
prompt_tokens_details, so message_delta carries resolved values.
|
||||
"""
|
||||
|
||||
async def sse_input():
|
||||
yield _make_stream_chunk(
|
||||
@@ -1296,7 +1301,8 @@ class TestStreamingCacheUsageSemantics:
|
||||
@pytest.mark.asyncio
|
||||
async def test_streaming_no_cache_hit(self):
|
||||
"""When the final chunk reports cached_tokens=0, message_delta carries
|
||||
cache fields = 0 (cache miss); message_start still omits them."""
|
||||
cache fields = 0 (cache miss); message_start still omits them.
|
||||
"""
|
||||
|
||||
async def sse_input():
|
||||
yield _make_stream_chunk(
|
||||
@@ -1337,7 +1343,8 @@ class TestStreamingCacheUsageSemantics:
|
||||
@pytest.mark.asyncio
|
||||
async def test_streaming_no_prompt_tokens_details_at_all(self):
|
||||
"""If --enable-prompt-tokens-details is off, no chunk carries cache
|
||||
info; both message_start and message_delta omit cache fields."""
|
||||
info; both message_start and message_delta omit cache fields.
|
||||
"""
|
||||
|
||||
async def sse_input():
|
||||
yield _make_stream_chunk(
|
||||
@@ -1458,7 +1465,8 @@ class TestMessagesFullConverter:
|
||||
class TestCacheSalt:
|
||||
def test_cache_salt_passed_through(self):
|
||||
"""cache_salt on the Anthropic request reaches the converted
|
||||
ChatCompletionRequest so prefix-cache isolation works via /v1/messages."""
|
||||
ChatCompletionRequest so prefix-cache isolation works via /v1/messages.
|
||||
"""
|
||||
request = _make_request(
|
||||
[{"role": "user", "content": "Hello"}],
|
||||
cache_salt="tenant-abc-secret-salt",
|
||||
@@ -1663,7 +1671,8 @@ class TestClientErrorResponses:
|
||||
@staticmethod
|
||||
def _conversion_error() -> ValidationError:
|
||||
"""A real pydantic ValidationError like the one ChatCompletionRequest
|
||||
construction raises when Anthropic input violates the OpenAI schema."""
|
||||
construction raises when Anthropic input violates the OpenAI schema.
|
||||
"""
|
||||
|
||||
class _StubRequest(BaseModel):
|
||||
stop: Annotated[list[str], Field(max_length=4)] | None = None
|
||||
@@ -1674,7 +1683,8 @@ class TestClientErrorResponses:
|
||||
|
||||
def test_validation_error_returns_bad_request(self):
|
||||
"""A pydantic ValidationError during Anthropic->OpenAI conversion is
|
||||
surfaced as a 400 BadRequestError, not a 500."""
|
||||
surfaced as a 400 BadRequestError, not a 500.
|
||||
"""
|
||||
handler = MagicMock(spec=AnthropicServingMessages)
|
||||
handler.create_messages.side_effect = self._conversion_error()
|
||||
|
||||
|
||||
@@ -87,7 +87,8 @@ def _make_chunk(
|
||||
citations: list[Any] | None = None,
|
||||
) -> str:
|
||||
"""Build the ``data: {...}\\n\\n`` SSE frame the production code
|
||||
consumes."""
|
||||
consumes.
|
||||
"""
|
||||
delta: dict[str, Any] = {}
|
||||
if role is not None:
|
||||
delta["role"] = role
|
||||
@@ -135,7 +136,8 @@ async def _stream_from(items: list[str]) -> AsyncGenerator[str, None]:
|
||||
|
||||
async def _drain(serving: CohereServingChatV2, items: list[str]) -> list[str]:
|
||||
"""Drive ``_chat_completion_stream_to_v2`` over ``items`` and collect
|
||||
the emitted SSE frames."""
|
||||
the emitted SSE frames.
|
||||
"""
|
||||
request = CohereChatV2Request(
|
||||
model="m", messages=[{"role": "user", "content": "hi"}], stream=True
|
||||
)
|
||||
|
||||
@@ -50,7 +50,8 @@ def update_admission_stats_worker(listen_address, sock, args, client_config):
|
||||
# importable by `multiprocessing.spawn` (no closures, no nesting).
|
||||
def defer_addresses_stub_worker(listen_address, sock, args, client_config):
|
||||
"""Bind ROUTER/PULL with a kernel-assigned port, report the actual
|
||||
endpoints back via the pipe, then exit."""
|
||||
endpoints back via the pipe, then exit.
|
||||
"""
|
||||
ctx = zmq.Context()
|
||||
try:
|
||||
in_sock = make_zmq_socket(
|
||||
@@ -357,7 +358,8 @@ def test_external_process_monitoring(api_server_args):
|
||||
def test_gather_actual_addresses_end_to_end():
|
||||
"""Each child binds ROUTER/PULL with a kernel-picked port and reports
|
||||
the bound endpoints back via its per-child pipe; the manager surfaces
|
||||
them via :py:meth:`gather_actual_addresses`."""
|
||||
them via :py:meth:`gather_actual_addresses`.
|
||||
"""
|
||||
host = "127.0.0.1"
|
||||
num_servers = 4
|
||||
|
||||
@@ -407,7 +409,8 @@ def test_gather_actual_addresses_end_to_end():
|
||||
@pytest.mark.timeout(30)
|
||||
def test_gather_actual_addresses_child_crash_before_report():
|
||||
"""A child that exits before sending its endpoints must surface a
|
||||
clear ``RuntimeError`` rather than hang or return ``None`` slots."""
|
||||
clear ``RuntimeError`` rather than hang or return ``None`` slots.
|
||||
"""
|
||||
host = "127.0.0.1"
|
||||
num_servers = 2
|
||||
placeholder_inputs = [
|
||||
@@ -444,7 +447,8 @@ def test_gather_actual_addresses_child_crash_before_report():
|
||||
|
||||
def test_rust_frontend_launch_log_redacts_credentials(monkeypatch, caplog):
|
||||
"""The Rust frontend command carries every non-default arg as JSON.
|
||||
Credentials must not reach the log line."""
|
||||
Credentials must not reach the log line.
|
||||
"""
|
||||
import subprocess as subprocess_mod
|
||||
|
||||
from vllm.entrypoints.launchers.cli_args import make_arg_parser
|
||||
|
||||
@@ -152,7 +152,7 @@ def test_multiple_valid_inputs(serve_parser):
|
||||
|
||||
### Tests for serve argument validation that run prior to loading
|
||||
def test_enable_auto_choice_passes_without_tool_call_parser(serve_parser):
|
||||
"""Ensure validation fails if tool choice is enabled with no call parser"""
|
||||
"""Ensure validation fails if tool choice is enabled with no call parser."""
|
||||
# If we enable-auto-tool-choice, explode with no tool-call-parser
|
||||
args = serve_parser.parse_args(args=["--enable-auto-tool-choice"])
|
||||
with pytest.raises(TypeError):
|
||||
@@ -160,7 +160,7 @@ def test_enable_auto_choice_passes_without_tool_call_parser(serve_parser):
|
||||
|
||||
|
||||
def test_enable_auto_choice_passes_with_tool_call_parser(serve_parser):
|
||||
"""Ensure validation passes with tool choice enabled with a call parser"""
|
||||
"""Ensure validation passes with tool choice enabled with a call parser."""
|
||||
args = serve_parser.parse_args(
|
||||
args=[
|
||||
"--enable-auto-tool-choice",
|
||||
@@ -172,7 +172,7 @@ def test_enable_auto_choice_passes_with_tool_call_parser(serve_parser):
|
||||
|
||||
|
||||
def test_enable_auto_choice_fails_with_enable_reasoning(serve_parser):
|
||||
"""Ensure validation fails if reasoning is enabled with auto tool choice"""
|
||||
"""Ensure validation fails if reasoning is enabled with auto tool choice."""
|
||||
args = serve_parser.parse_args(
|
||||
args=[
|
||||
"--enable-auto-tool-choice",
|
||||
@@ -186,7 +186,8 @@ def test_enable_auto_choice_fails_with_enable_reasoning(serve_parser):
|
||||
|
||||
def test_passes_with_reasoning_parser(serve_parser):
|
||||
"""Ensure validation passes if reasoning is enabled
|
||||
with a reasoning parser"""
|
||||
with a reasoning parser
|
||||
"""
|
||||
args = serve_parser.parse_args(
|
||||
args=[
|
||||
"--reasoning-parser",
|
||||
@@ -197,7 +198,7 @@ def test_passes_with_reasoning_parser(serve_parser):
|
||||
|
||||
|
||||
def test_chat_template_validation_for_happy_paths(serve_parser):
|
||||
"""Ensure validation passes if the chat template exists"""
|
||||
"""Ensure validation passes if the chat template exists."""
|
||||
args = serve_parser.parse_args(
|
||||
args=["--chat-template", CHATML_JINJA_PATH.absolute().as_posix()]
|
||||
)
|
||||
@@ -205,7 +206,7 @@ def test_chat_template_validation_for_happy_paths(serve_parser):
|
||||
|
||||
|
||||
def test_chat_template_validation_for_sad_paths(serve_parser):
|
||||
"""Ensure validation fails if the chat template doesn't exist"""
|
||||
"""Ensure validation fails if the chat template doesn't exist."""
|
||||
args = serve_parser.parse_args(args=["--chat-template", "does/not/exist"])
|
||||
with pytest.raises(VLLMValidationError):
|
||||
validate_parsed_serve_args(args)
|
||||
@@ -304,13 +305,13 @@ def test_sse_keep_alive_interval_non_integer(serve_parser, value):
|
||||
],
|
||||
)
|
||||
def test_middleware(serve_parser, cli_args, expected_middleware):
|
||||
"""Ensure multiple middleware args are parsed properly"""
|
||||
"""Ensure multiple middleware args are parsed properly."""
|
||||
args = serve_parser.parse_args(args=cli_args)
|
||||
assert args.middleware == expected_middleware
|
||||
|
||||
|
||||
def test_default_chat_template_kwargs_parsing(serve_parser):
|
||||
"""Ensure default_chat_template_kwargs JSON is parsed correctly"""
|
||||
"""Ensure default_chat_template_kwargs JSON is parsed correctly."""
|
||||
args = serve_parser.parse_args(
|
||||
args=["--default-chat-template-kwargs", '{"enable_thinking": false}']
|
||||
)
|
||||
@@ -318,7 +319,7 @@ def test_default_chat_template_kwargs_parsing(serve_parser):
|
||||
|
||||
|
||||
def test_default_chat_template_kwargs_complex(serve_parser):
|
||||
"""Ensure complex default_chat_template_kwargs JSON is parsed correctly"""
|
||||
"""Ensure complex default_chat_template_kwargs JSON is parsed correctly."""
|
||||
kwargs_json = '{"enable_thinking": false, "custom_param": "value", "num": 42}'
|
||||
args = serve_parser.parse_args(args=["--default-chat-template-kwargs", kwargs_json])
|
||||
assert args.default_chat_template_kwargs == {
|
||||
@@ -329,7 +330,7 @@ def test_default_chat_template_kwargs_complex(serve_parser):
|
||||
|
||||
|
||||
def test_default_chat_template_kwargs_default_none(serve_parser):
|
||||
"""Ensure default_chat_template_kwargs defaults to None"""
|
||||
"""Ensure default_chat_template_kwargs defaults to None."""
|
||||
args = serve_parser.parse_args(args=[])
|
||||
assert args.default_chat_template_kwargs is None
|
||||
|
||||
@@ -355,7 +356,7 @@ def test_resolve_default_chat_template_kwargs(default_kwargs, cohere_format, exp
|
||||
|
||||
|
||||
def test_default_chat_template_kwargs_invalid_json(serve_parser):
|
||||
"""Ensure invalid JSON raises an error"""
|
||||
"""Ensure invalid JSON raises an error."""
|
||||
with pytest.raises(SystemExit):
|
||||
serve_parser.parse_args(
|
||||
args=["--default-chat-template-kwargs", "not valid json"]
|
||||
@@ -381,7 +382,8 @@ def test_default_chat_template_kwargs_invalid_json(serve_parser):
|
||||
)
|
||||
def test_served_model_name_parsing(tmp_path, vllm_parser, args, raises):
|
||||
"""Ensure that users don't misuse --served-model-name and end up with the default
|
||||
model tag instead of the one they intended to serve."""
|
||||
model tag instead of the one they intended to serve.
|
||||
"""
|
||||
# Call the serve subparser
|
||||
args.insert(0, "serve")
|
||||
# Create a dummy config file if the test case includes it
|
||||
@@ -402,7 +404,7 @@ def test_served_model_name_parsing(tmp_path, vllm_parser, args, raises):
|
||||
|
||||
### Tests for LoRA target modules parsing
|
||||
def test_lora_target_modules_single(serve_parser):
|
||||
"""Test parsing single lora-target-modules argument"""
|
||||
"""Test parsing single lora-target-modules argument."""
|
||||
args = serve_parser.parse_args(
|
||||
args=["--enable-lora", "--lora-target-modules", "o_proj"]
|
||||
)
|
||||
@@ -410,7 +412,7 @@ def test_lora_target_modules_single(serve_parser):
|
||||
|
||||
|
||||
def test_lora_target_modules_multiple(serve_parser):
|
||||
"""Test parsing multiple lora-target-modules arguments"""
|
||||
"""Test parsing multiple lora-target-modules arguments."""
|
||||
args = serve_parser.parse_args(
|
||||
args=[
|
||||
"--enable-lora",
|
||||
@@ -424,6 +426,6 @@ def test_lora_target_modules_multiple(serve_parser):
|
||||
|
||||
|
||||
def test_lora_target_modules_default_none(serve_parser):
|
||||
"""Test that lora-target-modules defaults to None"""
|
||||
"""Test that lora-target-modules defaults to None."""
|
||||
args = serve_parser.parse_args(args=[])
|
||||
assert args.lora_target_modules is None
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
Tests for DPSupervisor: unit tests and lifecycle integration tests.
|
||||
"""Tests for DPSupervisor: unit tests and lifecycle integration tests.
|
||||
|
||||
Lifecycle integration tests replace child vLLM servers with lightweight
|
||||
aiohttp "fake" servers controlled by the test, so the suite runs without GPUs.
|
||||
@@ -463,8 +462,7 @@ async def test_start_server_no_log_config_when_no_filter(monkeypatch):
|
||||
|
||||
|
||||
class MockVLLMServer:
|
||||
"""
|
||||
Minimal FastAPI server that mimics one vLLM replica.
|
||||
"""Minimal FastAPI server that mimics one vLLM replica.
|
||||
GET /health returns 200 when healthy, 503 otherwise.
|
||||
Health state is toggled by the test via set_healthy().
|
||||
"""
|
||||
@@ -582,8 +580,7 @@ def launch_mock_vllm_with_drain(
|
||||
|
||||
|
||||
async def _poll_supervisor_health(expected_status: int, use_ssl: bool = False) -> bool:
|
||||
"""
|
||||
GET /health on the supervisor once and check for expected_status.
|
||||
"""GET /health on the supervisor once and check for expected_status.
|
||||
|
||||
Pass expected_status=-1 to assert the supervisor is not listening yet
|
||||
(a connection error is expected). The supervisor only starts its HTTP
|
||||
@@ -705,8 +702,7 @@ async def _run_supervisor(
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_basic_lifecycle(monkeypatch):
|
||||
"""
|
||||
A) Supervisor is not listening while children are unhealthy.
|
||||
"""A) Supervisor is not listening while children are unhealthy.
|
||||
B) /health returns 200 once every child reports healthy.
|
||||
C) SIGTERM and shutdown
|
||||
"""
|
||||
@@ -777,8 +773,7 @@ async def test_basic_lifecycle_with_ssl(monkeypatch):
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failed_startup(monkeypatch):
|
||||
"""
|
||||
A) One of the vLLM servers crashes during startup.
|
||||
"""A) One of the vLLM servers crashes during startup.
|
||||
B) DPSupervisor detects this, and cleans up resources.
|
||||
"""
|
||||
args = _make_args()
|
||||
@@ -801,8 +796,7 @@ async def test_failed_startup(monkeypatch):
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_becomes_unhealthy(monkeypatch):
|
||||
"""
|
||||
A) Supervisor is not listening while children are unhealthy.
|
||||
"""A) Supervisor is not listening while children are unhealthy.
|
||||
B) /health returns 200 once every child reports healthy.
|
||||
C) Child process becomes unhealthy.
|
||||
D) Detected and shutdown.
|
||||
@@ -842,8 +836,7 @@ async def test_becomes_unhealthy(monkeypatch):
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dp_server_fails(monkeypatch):
|
||||
"""
|
||||
A) Supervisor is not listening while children are unhealthy.
|
||||
"""A) Supervisor is not listening while children are unhealthy.
|
||||
B) /health returns 200 once every child reports healthy.
|
||||
C) Child process fails.
|
||||
D) Detected and shutdown.
|
||||
@@ -886,8 +879,7 @@ async def test_dp_server_fails(monkeypatch):
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_shutdown_timeout(monkeypatch: pytest.MonkeyPatch):
|
||||
"""
|
||||
Child mock servers delay shutdown by 10s on SIGTERM (simulating in-flight
|
||||
"""Child mock servers delay shutdown by 10s on SIGTERM (simulating in-flight
|
||||
request drain). The supervisor is configured with shutdown_timeout=10,
|
||||
so its total wait budget is 10 + CHILD_EXIT_GRACE_S seconds. The
|
||||
children exit naturally within that window, so no force-kill should occur
|
||||
|
||||
@@ -509,9 +509,7 @@ def test_completions():
|
||||
|
||||
|
||||
def test_completions_invalid_input():
|
||||
"""
|
||||
Ensure that we fail when the input doesn't conform to the openai api.
|
||||
"""
|
||||
"""Ensure that we fail when the input doesn't conform to the openai api."""
|
||||
with (
|
||||
tempfile.NamedTemporaryFile("w") as input_file,
|
||||
tempfile.NamedTemporaryFile("r") as output_file,
|
||||
@@ -602,9 +600,7 @@ def test_score(input_batch):
|
||||
|
||||
|
||||
def test_reasoning_parser():
|
||||
"""
|
||||
Test that reasoning_parser parameter works correctly in run_batch.
|
||||
"""
|
||||
"""Test that reasoning_parser parameter works correctly in run_batch."""
|
||||
with (
|
||||
tempfile.NamedTemporaryFile("w") as input_file,
|
||||
tempfile.NamedTemporaryFile("r") as output_file,
|
||||
@@ -767,8 +763,7 @@ def test_translation():
|
||||
|
||||
|
||||
def test_tool_calling():
|
||||
"""
|
||||
Test that tool calling works correctly in run_batch.
|
||||
"""Test that tool calling works correctly in run_batch.
|
||||
Verifies that requests with tools return tool_calls in the response.
|
||||
"""
|
||||
with (
|
||||
|
||||
@@ -134,7 +134,6 @@ async def test_shutdown_on_engine_failure(tmp_path: Path):
|
||||
Starts a vLLM server, kills it to simulate a crash, then verifies that
|
||||
subsequent API calls fail appropriately.
|
||||
"""
|
||||
|
||||
port = get_open_port()
|
||||
|
||||
# Redirect server output to a file rather than a pipe: nothing drains the
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for HF_HUB_OFFLINE mode"""
|
||||
"""Tests for HF_HUB_OFFLINE mode."""
|
||||
|
||||
import importlib
|
||||
import sys
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
This file test accuracy of the vLLM server via LMEval.
|
||||
"""This file test accuracy of the vLLM server via LMEval.
|
||||
It uses local-completions, which interacts with vLLM
|
||||
through the OAI API with N concurrent connections.
|
||||
This simulates real work usage of the API and makes
|
||||
@@ -33,7 +32,6 @@ EXPECTED_VALUES = {
|
||||
|
||||
def run_test(model_name, more_args=None):
|
||||
"""Run the end to end accuracy test."""
|
||||
|
||||
model_args = f"pretrained={model_name},max_model_len=4096"
|
||||
|
||||
if more_args is not None:
|
||||
@@ -64,7 +62,6 @@ TPU_TP_TEST_STR = "" # "tensor_parallel_size=4"
|
||||
@pytest.mark.parametrize("model", MODEL_NAMES)
|
||||
def test_lm_eval_accuracy_v1_engine(model):
|
||||
"""Run with the V1 Engine."""
|
||||
|
||||
more_args = None
|
||||
if current_platform.is_tpu():
|
||||
# Limit compilation time for TPU V1
|
||||
@@ -81,7 +78,6 @@ def test_lm_eval_accuracy_v1_engine(model):
|
||||
@pytest.mark.parametrize("model", FP8_KV_MODEL_NAMES)
|
||||
def test_lm_eval_accuracy_v1_engine_fp8_kv_cache(model):
|
||||
"""Run with the V1 Engine."""
|
||||
|
||||
more_args = None
|
||||
if current_platform.is_tpu():
|
||||
# Limit compilation time for TPU V1
|
||||
|
||||
@@ -20,8 +20,7 @@ def text_llm(vllm_runner):
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def llm_for_failure_test(vllm_runner):
|
||||
"""
|
||||
Fixture for testing issue #26081.
|
||||
"""Fixture for testing issue #26081.
|
||||
Uses a small max_model_len to easily trigger length errors.
|
||||
"""
|
||||
with vllm_runner(
|
||||
@@ -67,8 +66,7 @@ def test_multi_chat(text_llm):
|
||||
|
||||
|
||||
def test_llm_chat_tokenization_no_double_bos(text_llm):
|
||||
"""
|
||||
LLM.chat() should not add special tokens when using chat templates.
|
||||
"""LLM.chat() should not add special tokens when using chat templates.
|
||||
Check we get a single BOS token for llama chat.
|
||||
"""
|
||||
messages = [
|
||||
@@ -127,8 +125,7 @@ def test_chat_extra_kwargs(thinking_llm, enable_thinking):
|
||||
|
||||
|
||||
def test_chat_batch_failure_cleanup(llm_for_failure_test):
|
||||
"""
|
||||
Tests that if a batch call to llm.chat() fails mid-way
|
||||
"""Tests that if a batch call to llm.chat() fails mid-way
|
||||
(e.g., due to one invalid prompt), the requests that
|
||||
were already enqueued are properly aborted and do not
|
||||
pollute the queue for subsequent calls.
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Test that ``InputProcessor.inject_into_mm_cache()`` correctly injects
|
||||
"""Test that ``InputProcessor.inject_into_mm_cache()`` correctly injects
|
||||
pre-processed mm_kwargs into the processor cache and reports MM cache
|
||||
hit rate metrics accurately.
|
||||
|
||||
|
||||
@@ -47,8 +47,7 @@ async def client(server):
|
||||
async def test_online_audio_in_video(
|
||||
client: openai.AsyncOpenAI, video_assets: VideoTestAssets
|
||||
):
|
||||
"""Test video input with `audio_in_video=True`"""
|
||||
|
||||
"""Test video input with `audio_in_video=True`."""
|
||||
# we don't use video_urls above because they missed audio stream.
|
||||
video_path = video_assets[0].video_path
|
||||
with open(video_path, "rb") as f:
|
||||
@@ -97,8 +96,7 @@ async def test_online_audio_in_video(
|
||||
async def test_online_audio_in_video_multi_videos(
|
||||
client: openai.AsyncOpenAI, video_assets: VideoTestAssets
|
||||
):
|
||||
"""Test multi-video input with `audio_in_video=True`"""
|
||||
|
||||
"""Test multi-video input with `audio_in_video=True`."""
|
||||
# we don't use video_urls above because they missed audio stream.
|
||||
video_path = video_assets[0].video_path
|
||||
with open(video_path, "rb") as f:
|
||||
@@ -151,8 +149,7 @@ async def test_online_audio_in_video_multi_videos(
|
||||
async def test_online_audio_in_video_interleaved(
|
||||
client: openai.AsyncOpenAI, video_assets: VideoTestAssets
|
||||
):
|
||||
"""Test interleaved video/audio input with `audio_in_video=True`"""
|
||||
|
||||
"""Test interleaved video/audio input with `audio_in_video=True`."""
|
||||
# we don't use video_urls above because they missed audio stream.
|
||||
video_path = video_assets[0].video_path
|
||||
with open(video_path, "rb") as f:
|
||||
|
||||
+2
-1
@@ -2,7 +2,8 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""E2E test for mixing `prompt_embeds` with `audio_embeds` in a single
|
||||
Chat Completions request."""
|
||||
Chat Completions request.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
|
||||
+2
-1
@@ -68,7 +68,8 @@ async def client(server):
|
||||
def image_url() -> str:
|
||||
"""Stable real image as a data URL, kept identical across both the
|
||||
text and prompt_embeds requests so any output difference must come from
|
||||
how the text content is delivered."""
|
||||
how the text content is delivered.
|
||||
"""
|
||||
return encode_image_url(ImageAsset("stop_sign").pil_image)
|
||||
|
||||
|
||||
|
||||
@@ -34,8 +34,7 @@ REQUIRED_BEAM_SEARCH_TERMS = [
|
||||
|
||||
|
||||
def check_output_matches_terms(content: str, term_groups: list[list[str]]) -> bool:
|
||||
"""
|
||||
Check if content matches all required term groups.
|
||||
"""Check if content matches all required term groups.
|
||||
Each term group requires at least one of its terms to be present.
|
||||
All term groups must be satisfied.
|
||||
"""
|
||||
@@ -146,7 +145,8 @@ def describe_image_messages(
|
||||
) -> list[dict]:
|
||||
"""Build the system + user messages used by the completions-with-image
|
||||
family of tests. *extra_image_fields* is merged into the top-level
|
||||
image content block (for uuid / bad-key tests)."""
|
||||
image content block (for uuid / bad-key tests).
|
||||
"""
|
||||
image_block: dict = {
|
||||
"type": "image_url",
|
||||
"image_url": {"url": image_url},
|
||||
@@ -176,7 +176,8 @@ async def complete_and_check(
|
||||
temperature: float = 0.0,
|
||||
) -> str:
|
||||
"""Run a chat completion and assert the output is non-empty.
|
||||
Returns the content string."""
|
||||
Returns the content string.
|
||||
"""
|
||||
chat_completion = await client.chat.completions.create(
|
||||
model=model_name,
|
||||
messages=messages,
|
||||
|
||||
@@ -259,7 +259,7 @@ async def _generator(prompt_idx: int, text: str) -> AsyncGenerator[RequestOutput
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skip_global_cleanup
|
||||
async def test_batched_echo_does_not_prepend_user_prompt() -> None:
|
||||
"""echo must only echo the assistant turn, never the user's prompt.
|
||||
"""Echo must only echo the assistant turn, never the user's prompt.
|
||||
|
||||
With `add_generation_prompt` (the default) the response role is `assistant`
|
||||
while the last conversation message is the user's, so there is nothing to
|
||||
|
||||
@@ -121,7 +121,7 @@ def _build_serving_chat(engine: AsyncLLM) -> OpenAIServingChat:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_error_non_stream():
|
||||
"""test finish_reason='error' returns 500 InternalServerError (non-streaming)"""
|
||||
"""Test finish_reason='error' returns 500 InternalServerError (non-streaming)."""
|
||||
mock_engine = MagicMock(spec=AsyncLLM)
|
||||
mock_engine.errored = False
|
||||
mock_engine.model_config = MockModelConfig()
|
||||
@@ -246,7 +246,7 @@ async def test_renderer_only_chat_request_skips_mm_cache():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_error_stream():
|
||||
"""test finish_reason='error' returns 500 InternalServerError (streaming)"""
|
||||
"""Test finish_reason='error' returns 500 InternalServerError (streaming)."""
|
||||
mock_engine = MagicMock(spec=AsyncLLM)
|
||||
mock_engine.errored = False
|
||||
mock_engine.model_config = MockModelConfig()
|
||||
@@ -465,7 +465,8 @@ def test_system_message_warns_on_video(video_content):
|
||||
def test_json_schema_response_format_missing_schema():
|
||||
"""When response_format type is 'json_schema' but the json_schema field
|
||||
is not provided, request construction should raise a validation error
|
||||
so the API returns 400 instead of 500."""
|
||||
so the API returns 400 instead of 500.
|
||||
"""
|
||||
with pytest.raises(Exception, match="json_schema.*must be provided"):
|
||||
ChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
@@ -519,7 +520,8 @@ def test_structured_outputs_structural_tag_invalid(structural_tag):
|
||||
@pytest.mark.parametrize("field_name", ["prompt_logprobs", "top_logprobs"])
|
||||
def test_non_numeric_logprobs_rejected(field_name):
|
||||
"""A non-numeric logprobs value must be a clean 400 validation error, not a
|
||||
TypeError from the mode='before' comparison (which surfaces as HTTP 500)."""
|
||||
TypeError from the mode='before' comparison (which surfaces as HTTP 500).
|
||||
"""
|
||||
with pytest.raises(VLLMValidationError, match=f"`{field_name}` must be an integer"):
|
||||
ChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
|
||||
@@ -81,7 +81,8 @@ async def test_chat_logit_bias_invalid(client):
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_logit_bias_non_integer_key(client):
|
||||
"""Test that a non-integer logit_bias key is rejected with a clean,
|
||||
informative error instead of a raw 'invalid literal for int()' message."""
|
||||
informative error instead of a raw 'invalid literal for int()' message.
|
||||
"""
|
||||
with pytest.raises(openai.BadRequestError) as excinfo:
|
||||
await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
@@ -101,7 +102,8 @@ async def test_chat_logit_bias_non_integer_key(client):
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_logit_bias_non_numeric_value(client):
|
||||
"""Test that a non-numeric logit_bias value is rejected with a message
|
||||
that names the specific offending token, not just a generic TypeError."""
|
||||
that names the specific offending token, not just a generic TypeError.
|
||||
"""
|
||||
with pytest.raises(openai.BadRequestError) as excinfo:
|
||||
await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
@@ -120,7 +122,8 @@ async def test_chat_logit_bias_non_numeric_value(client):
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_logit_bias_multiple_non_integer_keys(client):
|
||||
"""Test that ALL invalid logit_bias keys are reported together,
|
||||
not just the first one encountered."""
|
||||
not just the first one encountered.
|
||||
"""
|
||||
with pytest.raises(openai.BadRequestError) as excinfo:
|
||||
await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
|
||||
@@ -451,7 +451,7 @@ async def test_inconsistent_tool_choice_and_tools(
|
||||
async def test_max_tokens_with_tool_choice_required(
|
||||
client: openai.AsyncOpenAI, tool_choice
|
||||
):
|
||||
""" """
|
||||
"""Requesting a single token with a required tool choice must not crash."""
|
||||
models = await client.models.list()
|
||||
model_name: str = models.data[0].id
|
||||
|
||||
|
||||
@@ -52,9 +52,10 @@ def stop_sign_image_url():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_translategemma_extra_lang_code_fields(client: openai.AsyncOpenAI):
|
||||
"""en -> es translation through TranslateGemma's bundled chat template's
|
||||
"""En -> es translation through TranslateGemma's bundled chat template's
|
||||
``text`` content branch, which depends on ``source_lang_code`` /
|
||||
``target_lang_code`` extra fields being preserved on the content part."""
|
||||
``target_lang_code`` extra fields being preserved on the content part.
|
||||
"""
|
||||
completion = await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
messages=[
|
||||
@@ -92,13 +93,14 @@ async def test_translategemma_image_extra_lang_code_fields(
|
||||
client: openai.AsyncOpenAI,
|
||||
stop_sign_image_url: str,
|
||||
):
|
||||
"""en -> es OCR-translation through TranslateGemma's bundled chat
|
||||
"""En -> es OCR-translation through TranslateGemma's bundled chat
|
||||
template's ``image`` content branch. Exercises the multimodal branch of
|
||||
``_collect_extra_fields`` in ``_parse_chat_message_content_part``: the
|
||||
``source_lang_code`` / ``target_lang_code`` extras must survive parsing
|
||||
and end up alongside the ``{"type": "image"}`` placeholder so the
|
||||
template's image branch (``content["type"] == 'image'``) renders the
|
||||
right OCR-translation prompt."""
|
||||
right OCR-translation prompt.
|
||||
"""
|
||||
completion = await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
messages=[
|
||||
|
||||
@@ -191,7 +191,8 @@ async def test_logprob_token_ids_stream_with_no_top_k(server):
|
||||
async def test_logprob_token_ids_default_behavior_unchanged(server):
|
||||
"""Without `logprob_token_ids`, the response carries the natural top-k
|
||||
most-likely tokens. This guards against the new field accidentally
|
||||
changing the default-path output."""
|
||||
changing the default-path output.
|
||||
"""
|
||||
async with server.get_async_client() as client:
|
||||
resp = await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
|
||||
@@ -1232,7 +1232,6 @@ async def test_serving_chat_mistral_token_ids_prompt_is_validated():
|
||||
"""Regression test: when the Mistral tokenizer path returns token IDs
|
||||
directly, we must still apply input length + max_tokens validation.
|
||||
"""
|
||||
|
||||
mock_engine = MagicMock(spec=AsyncLLM)
|
||||
mock_engine.errored = False
|
||||
mock_engine.model_config = MockModelConfig(skip_tokenizer_init=True)
|
||||
@@ -1271,7 +1270,6 @@ async def test_serving_chat_mistral_token_ids_prompt_too_long_is_rejected():
|
||||
"""Regression test: MistralTokenizer token-id prompts must still enforce
|
||||
the max context length for the input itself (token_num >= max_model_len).
|
||||
"""
|
||||
|
||||
mock_engine = MagicMock(spec=AsyncLLM)
|
||||
mock_engine.errored = False
|
||||
mock_engine.model_config = MockModelConfig(skip_tokenizer_init=True)
|
||||
@@ -1408,7 +1406,8 @@ async def test_serving_chat_did_set_correct_cache_salt(model_type):
|
||||
@pytest.mark.asyncio
|
||||
async def test_serving_chat_data_parallel_rank_extraction():
|
||||
"""Test that data_parallel_rank is properly extracted from header and
|
||||
passed to engine."""
|
||||
passed to engine.
|
||||
"""
|
||||
mock_engine = MagicMock(spec=AsyncLLM)
|
||||
mock_engine.errored = False
|
||||
mock_engine.model_config = MockModelConfig()
|
||||
@@ -1512,8 +1511,7 @@ async def _render_chat_prompt_token_ids(
|
||||
|
||||
|
||||
class TestServingChatWithHarmony:
|
||||
"""
|
||||
These tests ensure Chat Completion requests are being properly converted into
|
||||
"""These tests ensure Chat Completion requests are being properly converted into
|
||||
Harmony messages and Harmony response messages back into Chat Completion responses.
|
||||
These tests are not exhaustive, but each one was created to cover a specific case
|
||||
that we got wrong but is now fixed.
|
||||
@@ -1731,7 +1729,8 @@ class TestServingChatWithHarmony:
|
||||
@pytest.mark.asyncio
|
||||
async def test_system_message_without_tools(self, serving_chat, stream):
|
||||
"""Leading system message produces a developer message with
|
||||
DeveloperContent (# Instructions header)."""
|
||||
DeveloperContent (# Instructions header).
|
||||
"""
|
||||
messages = [
|
||||
{"role": "system", "content": "You are a helpful assistant."},
|
||||
{"role": "user", "content": "Hello"},
|
||||
@@ -1753,7 +1752,8 @@ class TestServingChatWithHarmony:
|
||||
@pytest.mark.asyncio
|
||||
async def test_system_message_with_tools(self, serving_chat, stream, weather_tools):
|
||||
"""Leading system message is folded into the developer message
|
||||
alongside tool definitions."""
|
||||
alongside tool definitions.
|
||||
"""
|
||||
messages = [
|
||||
{"role": "system", "content": "You are a helpful assistant."},
|
||||
{"role": "user", "content": "What's the weather?"},
|
||||
@@ -2168,7 +2168,8 @@ class TestServingChatWithHarmony:
|
||||
@pytest.mark.asyncio
|
||||
async def test_tool_choice_validation_without_parser():
|
||||
"""Test that tool_choice='required' or named tool without tool_parser
|
||||
returns an appropriate error message."""
|
||||
returns an appropriate error message.
|
||||
"""
|
||||
mock_engine = MagicMock(spec=AsyncLLM)
|
||||
mock_engine.errored = False
|
||||
mock_engine.model_config = MockModelConfig()
|
||||
|
||||
@@ -33,7 +33,6 @@ def _count_reasoning_decode_token_ids_between_markers(
|
||||
reasoning_end_ids: list[int],
|
||||
) -> int | None:
|
||||
"""Count decode tokens in the thinking span (after last start, before first end)."""
|
||||
|
||||
if not reasoning_start_ids or not reasoning_end_ids:
|
||||
raise ValueError("reasoning marker token id lists must be non-empty")
|
||||
|
||||
@@ -152,8 +151,8 @@ async def client(request, server, server_with_auto_reasoning_config):
|
||||
@pytest.mark.parametrize("client", ["default", "auto_config"], indirect=True)
|
||||
async def test_thinking_token_budget_mixed_requests(client: openai.AsyncOpenAI):
|
||||
"""Test that mixed requests (some with thinking_token_budget, some without)
|
||||
complete successfully without errors."""
|
||||
|
||||
complete successfully without errors.
|
||||
"""
|
||||
response_with_budget = await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
messages=MESSAGES,
|
||||
@@ -182,7 +181,6 @@ async def test_thinking_token_budget_limits_reasoning(client: openai.AsyncOpenAI
|
||||
grouped into streamed chunks (a single chunk can carry several tokens under
|
||||
async scheduling / stream_interval > 1). Counting chunks under-counts.
|
||||
"""
|
||||
|
||||
tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
|
||||
start_ids = list(tokenizer.encode(REASONING_START_STR, add_special_tokens=False))
|
||||
end_ids = list(tokenizer.encode(REASONING_END_STR, add_special_tokens=False))
|
||||
@@ -227,7 +225,6 @@ async def test_thinking_token_budget_qwen35_fp8_mtp_concurrent_mixed_budget_and_
|
||||
Qwen3.5 FP8 + MTP (TP=2) server. Budgeted calls are checked with
|
||||
``_count_reasoning_decode_token_ids_between_markers`` on full token ids.
|
||||
"""
|
||||
|
||||
_batch_spec: list[tuple[Literal["budget"], int] | tuple[Literal["plain"], None]] = [
|
||||
("budget", 1),
|
||||
("budget", 12),
|
||||
|
||||
@@ -350,7 +350,6 @@ async def test_parallel_no_streaming(client: openai.AsyncOpenAI, model_name: str
|
||||
"""Parallel sampling without streaming.
|
||||
A single request output contains a list of completions.
|
||||
"""
|
||||
|
||||
prompt = "What is an LLM?"
|
||||
n = 3
|
||||
max_tokens = 50
|
||||
@@ -396,7 +395,6 @@ async def test_parallel_streaming(client: openai.AsyncOpenAI, model_name: str):
|
||||
The tokens from multiple samples, are flattened into a single stream,
|
||||
with an index to indicate which sample the token belongs to.
|
||||
"""
|
||||
|
||||
prompt = "What is an LLM?"
|
||||
n = 3
|
||||
max_tokens = 50
|
||||
|
||||
@@ -303,7 +303,7 @@ def test_completion_metrics_carries_both_timing_and_spec_decode():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_completion_error_non_stream():
|
||||
"""test finish_reason='error' returns 500 InternalServerError (non-streaming)"""
|
||||
"""Test finish_reason='error' returns 500 InternalServerError (non-streaming)."""
|
||||
mock_engine = MagicMock(spec=AsyncLLM)
|
||||
mock_engine.errored = False
|
||||
mock_engine.model_config = MockModelConfig()
|
||||
@@ -439,7 +439,7 @@ async def test_renderer_only_completion_request_skips_mm_cache():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_completion_error_stream():
|
||||
"""test finish_reason='error' returns 500 InternalServerError (streaming)"""
|
||||
"""Test finish_reason='error' returns 500 InternalServerError (streaming)."""
|
||||
mock_engine = MagicMock(spec=AsyncLLM)
|
||||
mock_engine.errored = False
|
||||
mock_engine.model_config = MockModelConfig()
|
||||
@@ -521,7 +521,8 @@ async def test_completion_error_stream():
|
||||
def test_json_schema_response_format_missing_schema():
|
||||
"""When response_format type is 'json_schema' but the json_schema field
|
||||
is not provided, request construction should raise a validation error
|
||||
so the API returns 400 instead of 500."""
|
||||
so the API returns 400 instead of 500.
|
||||
"""
|
||||
with pytest.raises(Exception, match="json_schema.*must be provided"):
|
||||
CompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
@@ -584,7 +585,8 @@ def test_negative_prompt_token_ids_flat():
|
||||
def test_logprobs_minus_one_allowed():
|
||||
"""logprobs=-1 means "return all logprobs". The sampling layer and the chat
|
||||
top_logprobs / prompt_logprobs validators all accept -1, so the completion
|
||||
logprobs validator must accept it too instead of rejecting it as negative."""
|
||||
logprobs validator must accept it too instead of rejecting it as negative.
|
||||
"""
|
||||
request = CompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
prompt="Test prompt",
|
||||
@@ -744,7 +746,8 @@ class TestCompletionPromptListLimit:
|
||||
@pytest.mark.parametrize("field_name", ["prompt_logprobs", "logprobs"])
|
||||
def test_non_numeric_logprobs_rejected(field_name):
|
||||
"""A non-numeric logprobs value must be a clean 400 validation error, not a
|
||||
TypeError from the mode='before' comparison (which surfaces as HTTP 500)."""
|
||||
TypeError from the mode='before' comparison (which surfaces as HTTP 500).
|
||||
"""
|
||||
with pytest.raises(VLLMValidationError, match=f"`{field_name}` must be an integer"):
|
||||
CompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
|
||||
@@ -39,8 +39,7 @@ def server():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_token_in_token_out_and_logprobs(server):
|
||||
"""
|
||||
Test token-in-token-out and token_ids align with prompt_logprobs
|
||||
"""Test token-in-token-out and token_ids align with prompt_logprobs
|
||||
& logprobs when return_tokens_as_token_ids is enabled.
|
||||
"""
|
||||
tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
This file test accuracy of the vLLM server via LMEval.
|
||||
"""This file test accuracy of the vLLM server via LMEval.
|
||||
It uses local-completions, which interacts with vLLM
|
||||
through the OAI API with N concurrent connections.
|
||||
This simulates real work usage of the API and makes
|
||||
@@ -37,7 +36,6 @@ if current_platform.is_tpu():
|
||||
|
||||
def run_test(more_args):
|
||||
"""Run the end to end accuracy test."""
|
||||
|
||||
args = list(DEFAULT_ARGS)
|
||||
args.extend(more_args)
|
||||
print(f"Running with: {args}")
|
||||
@@ -68,7 +66,6 @@ def run_test(more_args):
|
||||
|
||||
def test_lm_eval_accuracy_v1_engine():
|
||||
"""Run with the V1 Engine."""
|
||||
|
||||
more_args = []
|
||||
|
||||
# Limit compilation time for V1 on TPU
|
||||
|
||||
@@ -43,7 +43,8 @@ def _system():
|
||||
|
||||
class TestResponseInputToHarmonyRenderParity:
|
||||
"""Each test drives the same conversation through both APIs and asserts
|
||||
identical Harmony messages and rendered token sequences."""
|
||||
identical Harmony messages and rendered token sequences.
|
||||
"""
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# Single-message cases
|
||||
@@ -51,7 +52,8 @@ class TestResponseInputToHarmonyRenderParity:
|
||||
|
||||
def test_developer_message(self):
|
||||
"""Both APIs must render developer messages identically using
|
||||
DeveloperContent (with the '# Instructions' header)."""
|
||||
DeveloperContent (with the '# Instructions' header).
|
||||
"""
|
||||
chat_msgs = parse_chat_input_to_harmony_message(
|
||||
{"role": "developer", "content": "Be concise."}
|
||||
)
|
||||
@@ -246,7 +248,8 @@ class TestResponseInputToHarmonyRenderParity:
|
||||
def test_reasoning_combined_with_function_call(self):
|
||||
"""Chat API packs reasoning + tool_calls into one dict; responses API
|
||||
represents them as two separate items. Both must produce the same two
|
||||
Harmony messages in the same order: analysis then commentary."""
|
||||
Harmony messages in the same order: analysis then commentary.
|
||||
"""
|
||||
chat_msgs = parse_chat_input_to_harmony_message(
|
||||
{
|
||||
"role": "assistant",
|
||||
@@ -529,7 +532,8 @@ class TestResponseInputToHarmonyRenderParity:
|
||||
def test_completed_turns_drop_reasoning(self):
|
||||
"""Validates that reasoning from completed turns is dropped, while
|
||||
reasoning from the current in-progress tool-call turn is preserved
|
||||
in both chat completions and responses previous_input_messages."""
|
||||
in both chat completions and responses previous_input_messages.
|
||||
"""
|
||||
first_turn_reasoning = "FIRST_TURN_REASONING"
|
||||
second_turn_reasoning = "SECOND_TURN_REASONING"
|
||||
|
||||
|
||||
@@ -170,7 +170,7 @@ class TestIsFunctionRecipientWithAllowedNames:
|
||||
"""Tests for is_function_recipient with allowed_function_tool_names."""
|
||||
|
||||
def test_prefixed_always_accepted(self):
|
||||
"""functions. prefix is always accepted regardless of allowed names."""
|
||||
"""Functions. prefix is always accepted regardless of allowed names."""
|
||||
fn_names = frozenset({"other_tool"})
|
||||
assert is_function_recipient("functions.get_weather", fn_names) is True
|
||||
|
||||
@@ -252,8 +252,7 @@ class TestExtractFunctionFromRecipient:
|
||||
|
||||
|
||||
class TestCommonParseInputToHarmonyMessage:
|
||||
"""
|
||||
Tests for scenarios that are common to both Chat Completion
|
||||
"""Tests for scenarios that are common to both Chat Completion
|
||||
parse_chat_input_to_harmony_message and Responses API
|
||||
response_previous_input_to_harmony functions.
|
||||
"""
|
||||
@@ -324,7 +323,8 @@ class TestCommonParseInputToHarmonyMessage:
|
||||
|
||||
def test_system_message(self, parse_function):
|
||||
"""Test parsing system messages, which are parsed into developer messages
|
||||
with DeveloperContent."""
|
||||
with DeveloperContent.
|
||||
"""
|
||||
chat_msg = {
|
||||
"role": "system",
|
||||
"content": "You are a helpful assistant",
|
||||
@@ -448,8 +448,7 @@ class TestCommonParseInputToHarmonyMessage:
|
||||
|
||||
|
||||
class TestParseChatInputToHarmonyMessage:
|
||||
"""
|
||||
Tests for scenarios that are specific to the Chat Completion API
|
||||
"""Tests for scenarios that are specific to the Chat Completion API
|
||||
parse_chat_input_to_harmony_message function.
|
||||
"""
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@ from vllm.exceptions import GenerationError
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_raise_if_error_raises_generation_error():
|
||||
"""test _raise_if_error raises GenerationError"""
|
||||
"""Test _raise_if_error raises GenerationError."""
|
||||
# create a minimal GenerateBaseServing instance
|
||||
mock_engine = MagicMock()
|
||||
mock_engine.model_config = MagicMock()
|
||||
@@ -42,7 +42,7 @@ async def test_raise_if_error_raises_generation_error():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_convert_generation_error_to_streaming_response():
|
||||
"""test _convert_generation_error_to_streaming_response output"""
|
||||
"""Test _convert_generation_error_to_streaming_response output."""
|
||||
mock_engine = MagicMock()
|
||||
mock_engine.model_config = MagicMock()
|
||||
mock_engine.model_config.max_model_len = 100
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user