Add pydocstyle to the ruff rules (#52136)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Signed-off-by: Harry Mellor <hej.mellor@gmail.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-09-16 16:44:29 +00:00
committed by GitHub
co-authored by Claude Opus 5
parent 804e5377aa
commit 8c1557a79c
2002 changed files with 15024 additions and 14269 deletions
@@ -151,6 +151,7 @@ def create_and_prepopulate_kv_cache(
Returns:
A 4D tensor in logical ``(num_blocks, num_kv_heads, block_size,
2 * head_size)`` order with strides determined by ``layout``.
"""
batch_size = len(k_contexts)
seq_lens = common_attn_metadata.seq_lens.cpu()
@@ -286,7 +287,6 @@ def run_attention_backend(
use_cuda_graph: bool = False,
) -> torch.Tensor:
"""Run attention computation using the specified backend's AttentionImpl."""
use_direct_block_mask = is_torch_equal_or_newer("2.9.0.dev0")
if backend == "FLEX_ATTENTION_SLOW":
use_direct_block_mask = False
@@ -414,8 +414,7 @@ def _test_backend_correctness(
max_num_seqs: int | None = None,
max_num_batched_tokens: int | None = None,
):
"""
Test that all backends produce similar outputs to a reference implementation
"""Test that all backends produce similar outputs to a reference implementation
using FlexAttention or an explicit attention-sink reference.
This test works by:
@@ -1393,7 +1392,8 @@ def test_non_causal_backend_correctness(
model: str,
):
"""Test backend's correctness with non-causal (bidirectional) decoder
attention, as used by DFlash speculative decoding."""
attention, as used by DFlash speculative decoding.
"""
def bidirectional_mask_mod(
b: torch.Tensor,
@@ -115,7 +115,8 @@ def test_mamba_layers_have_unified_interface(
layer_class, expected_backend, expected_mamba_type
):
"""Test that all Mamba layers have the unified get_attn_backend
interface."""
interface.
"""
assert hasattr(layer_class, "get_attn_backend"), (
f"{layer_class.__name__} should have get_attn_backend method"
)
+16 -15
View File
@@ -20,12 +20,12 @@ from vllm.v1.worker.ubatch_utils import (
@pytest.fixture
def sample_query_start_loc():
"""Sample query_start_loc tensor for testing"""
"""Sample query_start_loc tensor for testing."""
return torch.tensor([0, 5, 12, 20, 35, 50])
def test_basic_slice_middle(sample_query_start_loc):
"""Test slicing from middle of tensor"""
"""Test slicing from middle of tensor."""
req_slice = slice(1, 3) # slice from index 1 to 3
result = slice_query_start_locs(sample_query_start_loc, req_slice)
@@ -34,7 +34,7 @@ def test_basic_slice_middle(sample_query_start_loc):
def test_slice_from_beginning(sample_query_start_loc):
"""Test slicing from the beginning of tensor"""
"""Test slicing from the beginning of tensor."""
req_slice = slice(0, 2) # slice from index 0 to 2
result = slice_query_start_locs(sample_query_start_loc, req_slice)
@@ -43,7 +43,7 @@ def test_slice_from_beginning(sample_query_start_loc):
def test_slice_to_end(sample_query_start_loc):
"""Test slicing to the end of tensor"""
"""Test slicing to the end of tensor."""
req_slice = slice(3, 5) # slice from index 3 to 5 (last index)
result = slice_query_start_locs(sample_query_start_loc, req_slice)
@@ -52,7 +52,7 @@ def test_slice_to_end(sample_query_start_loc):
def test_single_element_slice(sample_query_start_loc):
"""Test slice that results in single element"""
"""Test slice that results in single element."""
req_slice = slice(2, 3) # slice from index 2 to 3
result = slice_query_start_locs(sample_query_start_loc, req_slice)
@@ -61,7 +61,7 @@ def test_single_element_slice(sample_query_start_loc):
def test_full_tensor_slice(sample_query_start_loc):
"""Test slicing the entire tensor"""
"""Test slicing the entire tensor."""
req_slice = slice(0, 5) # slice entire tensor
result = slice_query_start_locs(sample_query_start_loc, req_slice)
@@ -80,7 +80,7 @@ def test_slice_bounds_edge_cases(sample_query_start_loc):
@pytest.fixture
def small_decode_metadata():
"""Create metadata for small decode batch"""
"""Create metadata for small decode batch."""
batch_spec = BATCH_SPECS["small_decode"]
device = torch.device("cpu")
return create_common_attn_metadata(batch_spec, block_size=16, device=device)
@@ -88,7 +88,7 @@ def small_decode_metadata():
@pytest.fixture
def large_decode_metadata():
"""Create metadata for small decode batch"""
"""Create metadata for small decode batch."""
batch_spec = BATCH_SPECS["large_decode"]
device = torch.device("cpu")
return create_common_attn_metadata(batch_spec, block_size=16, device=device)
@@ -96,7 +96,7 @@ def large_decode_metadata():
@pytest.fixture
def mixed_small_metadata():
"""Create metadata for mixed small batch"""
"""Create metadata for mixed small batch."""
batch_spec = BATCH_SPECS["mixed_small"]
device = torch.device("cpu")
return create_common_attn_metadata(batch_spec, block_size=16, device=device)
@@ -104,7 +104,7 @@ def mixed_small_metadata():
# Tests for _make_metadata_with_slice
def test_make_metadata_with_slice_decode_batch(small_decode_metadata):
"""Test slicing decode batch metadata"""
"""Test slicing decode batch metadata."""
# Split first request only
ubatch_slice = UBatchSlice(slice(0, 1), slice(0, 1))
@@ -119,7 +119,7 @@ def test_make_metadata_with_slice_decode_batch(small_decode_metadata):
def test_make_metadata_with_slice_mixed_batch(mixed_small_metadata):
"""Test slicing mixed batch metadata"""
"""Test slicing mixed batch metadata."""
ubatch_slice = UBatchSlice(slice(1, 3), slice(1, 7)) # Requests 1-3, tokens 1-7
result = _make_metadata_with_slice(ubatch_slice, mixed_small_metadata)
@@ -132,7 +132,7 @@ def test_make_metadata_with_slice_mixed_batch(mixed_small_metadata):
def test_split_attn_metadata_decode_batch(large_decode_metadata):
"""Test splitting decode batch into two equal parts"""
"""Test splitting decode batch into two equal parts."""
num_tokens = large_decode_metadata.num_reqs
mid_point = num_tokens // 2
ubatch_slices = [
@@ -164,7 +164,8 @@ def apply_split_decodes_and_prefills(
treat_short_extends_as_decodes: bool = True,
):
"""Helper function to apply split_decodes_and_prefills and return
the results."""
the results.
"""
device = torch.device("cpu")
seq_lens = [10 * (i + 1) for i in range(len(query_lens))]
common_metadata = create_common_attn_metadata(
@@ -297,7 +298,7 @@ def test_split_decodes_and_prefills_uniform_mixed_batch_non_uniform_decodes():
def test_split_decodes_and_prefills_uniform_padded_batch_all_same():
"""uniform batch where all query lengths are identical with 0 length padded reqs."""
"""Uniform batch where all query lengths are identical with 0 length padded reqs."""
# All query lengths are 2, with decode_threshold=3 (so 2 <= 3)
# This triggers the padded uniform path at line 891
query_lens = [2, 2, 2, 0]
@@ -324,7 +325,7 @@ def test_split_decodes_and_prefills_uniform_padded_batch_all_same():
def test_prefill_split_across_ubatches(
seq_lens, query_lens, split_point, expected_first_reqs, expected_second_reqs
):
"""Test splitting a prefill across ubatches"""
"""Test splitting a prefill across ubatches."""
import numpy as np
device = torch.device("cpu")
+2 -1
View File
@@ -1,7 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""The empty-shard mask fused into the DCP A2A pack kernel must be bit-exact
with the eager ``mask_dcp_empty_shards_`` pass it replaces."""
with the eager ``mask_dcp_empty_shards_`` pass it replaces.
"""
import pytest
import torch
@@ -495,7 +495,8 @@ def test_dspark_noncausal_sparse_mla_matches_sdpa(
num_heads,
):
"""Non-causal (window ∪ block, future-pointing) per-token indices must match
a dense SDPA reference over the same indices, for both sparse-MLA backends."""
a dense SDPA reference over the same indices, for both sparse-MLA backends.
"""
_skip_if_backend_unavailable(backend_cls, kv_cache_dtype, block_size)
window, block, topk_width = _DSPARK_CONFIGS[config_name]
@@ -231,7 +231,8 @@ def test_plan_uses_state_params(monkeypatch):
def test_kv_lens_host_formula():
"""Per-row host lengths: context == position + 1; capped at
index_topk + trailing-pool remainder past the sparse threshold."""
index_topk + trailing-pool remainder past the sparse threshold.
"""
builder = object.__new__(FlashInferMLASparseSM90Builder)
builder._index_topk = 2048
builder._index_kpool = 4
@@ -187,7 +187,8 @@ def test_gdn_build_classification(test_case: GDNBuildTestCase):
def test_has_initial_state_after_reclassification():
"""After reclassification, num_prefills > 0 so the prefill kernel path
should compute has_initial_state. For the reclassified request with
context_lens > 0, the corresponding entry must be True."""
context_lens > 0, the corresponding entry must be True.
"""
builder = _create_gdn_builder(num_speculative_tokens=2)
batch = BatchSpec(seq_lens=[65, 20], query_lens=[1, 3])
meta = _build(builder, batch, num_decode_draft_tokens=[-1, 2])
@@ -56,7 +56,8 @@ def _ref_stable_topk_from_candidates_fp64(
) -> torch.Tensor:
"""Pure-PyTorch reference for the CuteDSL stable-topk selector order
(score desc, then lowest global token id). Selects the same SET as the
kernel; only the set is compared in tests."""
kernel; only the set is compared in tests.
"""
num_rows, num_candidates = candidate_scores.shape
device = candidate_scores.device
select_k = min(k, num_candidates)
@@ -121,7 +122,8 @@ def _dcp_lse_merge(
class _FakeDCPGroup:
"""Single-process stand-in: ``all_gather`` returns the pre-built
concatenation of every rank's packed ``(score, global_id)`` candidates,
mirroring the one packed all-gather the merge issues."""
mirroring the one packed all-gather the merge issues.
"""
def __init__(self, gathered_packed: torch.Tensor) -> None:
self.gathered_packed = gathered_packed
@@ -207,7 +209,8 @@ def _merge_local_topks_global_with_fake_dcp(
"""Run ``_merge_dcp_topk_global`` for every rank against a faked
all-gather and return each rank's global-index result (all ranks should
agree). The fake pre-builds the packed candidate concatenation the merge's
single ``all_gather`` would return."""
single ``all_gather`` would return.
"""
# The merge is now CuteDSL-only (no PyTorch fallback), so it runs the real
# Triton pack + CuteDSL selector kernels even behind the faked all-gather.
if not current_platform.is_cuda() or not has_cutedsl():
@@ -709,7 +712,8 @@ def test_dcp_filter_compaction_matches_reference(
"""In-kernel compaction must, for every row, produce exactly the rank-owned
physical slots packed into [0, valid_count) with -1 in the tail -- the same
SET a reference filter + sort/gather produces. Uses wide rows (> BLOCK_N
valid slots), with interior -1 gaps."""
valid slots), with interior -1 gaps.
"""
device = torch.device("cuda")
torch.manual_seed(7)
dcp_size = 2
@@ -867,7 +871,8 @@ def test_decode_topk_pads_surplus_with_negative_one():
so a non-(-1) pad would be silently attended. This is the common case under
DCP, where each rank's local seq_len = global / world is usually << topk.
Checked on the *set* of valid indices (the kernel may order them by score,
not position)."""
not position).
"""
torch.manual_seed(0)
device = torch.device("cuda")
topk = 8
@@ -913,7 +918,8 @@ def test_sparse_decode_dcp_short_context_matches_non_dcp():
"""End-to-end DCP decode where the global seq_len < topk (so every rank's
local top-k is surplus-padded). Exercises the kernel surplus -> merge mask
-> global top-k -> physical localize -> LSE merge chain for the common
short-context decode case, vs the non-DCP reference."""
short-context decode case, vs the non-DCP reference.
"""
torch.manual_seed(4)
device = torch.device("cuda")
world = 2
@@ -58,7 +58,8 @@ def test_tail_backend_layout_matches_kernel_pointer_arithmetic():
def make_tail_block_table(own_blocks, width=64):
"""Tail-group block table as BlockTables produces it: column 0 holds the
request's single KpoolTailManager block, the remaining columns are never
written and stay zero."""
written and stay zero.
"""
bt = torch.zeros(len(own_blocks), width, dtype=torch.int32)
bt[:, 0] = torch.tensor(own_blocks, dtype=torch.int32)
return bt
@@ -66,7 +67,8 @@ def make_tail_block_table(own_blocks, width=64):
def legacy_generic_tail_slots(block_table, query_start_loc, positions):
"""Reference of the generic ``_compute_slot_mappings_kernel`` arithmetic
(block_table.py:305-313) applied to the tail group's table."""
(block_table.py:305-313) applied to the tail group's table.
"""
slots = []
for req in range(block_table.shape[0]):
for i in range(query_start_loc[req], query_start_loc[req + 1]):
@@ -107,7 +109,8 @@ def make_batch(per_req_positions, padded_len=None):
def test_legacy_generic_mapping_collapses_onto_block_zero():
"""The bug: with the manager's 1-column block table, the generic kernel
maps every pos >= kpool onto tail block 0, and distinct requests collide."""
maps every pos >= kpool onto tail block 0, and distinct requests collide.
"""
own_blocks = [5, 9]
per_req = [list(range(10)), list(range(12))] # prompts of len 10 and 12
positions, qsl, _, num_actual, num_reqs = make_batch(per_req)
@@ -137,7 +140,8 @@ def test_legacy_generic_mapping_collapses_onto_block_zero():
def test_circular_mapping_isolates_requests():
"""The fix: every token lands in its own request's block at pos % kpool,
and no slot is ever shared by two different requests (slots do recur
within a request every kpool positions -- that is the circular design)."""
within a request every kpool positions -- that is the circular design).
"""
own_blocks = [5, 9]
per_req = [list(range(10)), list(range(12))]
positions, qsl, slot_mapping, num_actual, num_reqs = make_batch(per_req)
@@ -165,7 +169,8 @@ def test_circular_mapping_isolates_requests():
def test_circular_mapping_matches_generic_for_short_requests(prompt_len):
"""For pos < kpool the generic kernel already picks the own block, so the
two mappings agree while every position fits the request's first block
(single-request behavior is unchanged)."""
(single-request behavior is unchanged).
"""
own_blocks = [7]
per_req = [list(range(prompt_len))]
positions, qsl, slot_mapping, num_actual, num_reqs = make_batch(per_req)
@@ -275,7 +280,8 @@ class TailRingMirror:
"""Mirror of _kpool_tail_seed_kernel / _kpool_decode_update_batched_kernel
addressing: block = tail_slot // kpool, ring offset = pos % kpool; a pool
completing at pos reads ring slots (pool_start + s) % kpool and uses the
current token's own K/score for the last member."""
current token's own K/score for the last member.
"""
def __init__(self, num_blocks, kpool=KPOOL):
self.kpool = kpool
@@ -318,7 +324,8 @@ def tail_slot_for(mapping, req, pos, own_block):
def run_scenario(mapping, interleave):
"""Requests A (block 5, prompt len 9) and B (block 9, prompt len 11)
decode concurrently; returns A's boundary pool [8, 9, 10, 11]."""
decode concurrently; returns A's boundary pool [8, 9, 10, 11].
"""
ring = TailRingMirror(num_blocks=16)
blocks = {"A": 5, "B": 9}
prompts = {"A": 9, "B": 11}
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Regression test for https://github.com/vllm-project/vllm/issues/34865
"""Regression test for https://github.com/vllm-project/vllm/issues/34865.
When multiple KV cache groups share the same MambaSpec (as in Nemotron
hybrid models), the metadata caching optimization reuses metadata from
@@ -29,7 +28,8 @@ def _make_vllm_config(
block_size: int | None = None,
):
"""Create a minimal mock VllmConfig with only the fields the builder
accesses, avoiding any model download / HF config inspection."""
accesses, avoiding any model download / HF config inspection.
"""
speculative_config = (
SimpleNamespace(
num_speculative_tokens=num_speculative_tokens,
@@ -75,8 +75,8 @@ def test_mamba_single_token_prompt_runs_as_prefill():
def test_update_block_table_copies_block_idx_to_persistent_buffers():
"""update_block_table() must write block_idx tensors to the current
builder's persistent buffers, not leave them pointing to a different
builder's buffers."""
builder's buffers.
"""
block_size = 16
max_model_len = 256
num_reqs = 4
@@ -180,8 +180,8 @@ def test_state_indices_tensor_d_includes_num_speculative_blocks():
bug 1: with mamba_cache_mode='all' and speculative decoding enabled,
the cudagraph buffer for state_indices_tensor_d must allocate the same
per-request column count as the runtime block table, which includes
num_speculative_blocks trailing scratch columns."""
num_speculative_blocks trailing scratch columns.
"""
block_size = 16
max_model_len = 256
max_num_seqs = 4
@@ -214,8 +214,8 @@ def test_block_idx_cudagraph_capture_padded_by_num_reqs():
bug 2: with mamba_cache_mode='all' and spec decode, _update_metadata_for
_cudagraph_capture must slice block_idx_last_{scheduled,computed}_token
by the request count (padded_bs == num_reqs), not by num_decode_tokens.
Past num_decodes, the slice must be zero-filled."""
Past num_decodes, the slice must be zero-filled.
"""
block_size = 16
max_model_len = 256
max_num_seqs = 8
@@ -295,7 +295,8 @@ def test_block_idx_prev_step_persistent_buffer_allocated():
block_idx_last_scheduled_token_prev_step as a persistent buffer with the
same shape as the existing block_idx_last_{scheduled,computed}_token
buffers, so cudagraph capture records a stable pointer for the prev-step
input anchor consumed by mamba_mixer2's input gather."""
input anchor consumed by mamba_mixer2's input gather.
"""
block_size = 16
max_model_len = 256
max_num_seqs = 8
@@ -323,7 +324,8 @@ def test_block_idx_prev_step_persistent_buffer_allocated():
def test_block_idx_prev_step_persistent_buffer_skipped_without_spec_decode():
"""Without spec decode, the prev-step buffer is unused and must not be
allocated — the input anchor reduces to last_computed_token."""
allocated — the input anchor reduces to last_computed_token.
"""
block_size = 16
max_model_len = 256
max_num_seqs = 8
@@ -347,7 +349,8 @@ def test_block_idx_prev_step_cudagraph_capture_uses_persistent_buffer():
"""_update_metadata_for_cudagraph_capture must copy the prev-step anchor
into the builder's persistent buffer (so cudagraph replay reads from the
same underlying memory), pad past num_decodes with zero, and return a
slice of the persistent buffer in the metadata."""
slice of the persistent buffer in the metadata.
"""
block_size = 16
max_model_len = 256
max_num_seqs = 8
+4 -5
View File
@@ -171,7 +171,8 @@ def test_glm5_flashinfer_masked_mha_routing(
def test_concat_k_nope_k_pe_matches_torch_cat(qk_rope_head_dim):
"""The K concat used by the MLA prefill context loop must equal torch.cat of
k_nope with the broadcast k_pe; with no RoPE part it returns k_nope itself
instead of allocating and copying."""
instead of allocating and copying.
"""
torch.manual_seed(0)
num_tokens, num_heads, qk_nope_head_dim = 5, 4, 256
k_nope = torch.randn(num_tokens, num_heads, qk_nope_head_dim, dtype=torch.bfloat16)
@@ -515,6 +516,7 @@ def create_and_prepopulate_kv_cache(
Returns:
MLA KV cache tensor
"""
batch_size = len(kv_c_contexts)
seq_lens = common_attn_metadata.seq_lens.cpu()
@@ -1307,7 +1309,6 @@ def run_attention_backend(
chunked_prefill_workspace_size: int | None = None,
) -> torch.Tensor:
"""Run attention computation using the specified backend's AttentionImpl."""
builder_cls, impl_cls = try_get_attention_backend(backend)
# Force the prefill backend selection (None means auto-select).
@@ -1434,8 +1435,7 @@ def _run_backend_correctness(
v_head_dim: int,
chunked_prefill_workspace_size: int | None = None,
):
"""
Test that all backends produce similar outputs to a reference implementation
"""Test that all backends produce similar outputs to a reference implementation
using torch.nn.functional.scaled_dot_product_attention.
This test works by:
@@ -1454,7 +1454,6 @@ def _run_backend_correctness(
multiple GPUs. This tests that backends work correctly with different
head counts.
"""
# Filter backends to those that support the requested kv_cache_dtype
backends_to_test = [
b
@@ -134,7 +134,8 @@ def test_flash_attn_registers_warmup_only_for_fa4(
def test_flash_attn_prefill_backend_signature_accepts_fused_kwargs():
"""run_prefill_new_tokens must accept out/output_scale so the direct
(non-**kwargs) call in forward_mha type- and runtime-checks."""
(non-**kwargs) call in forward_mha type- and runtime-checks.
"""
import inspect
params = inspect.signature(
@@ -150,7 +151,8 @@ def test_flash_attn_prefill_backend_signature_accepts_fused_kwargs():
def test_mla_impl_forward_mha_accepts_output_scale():
"""The abstract MLA impl forward_mha must carry output_scale so every
override (and the unconditional forward_impl call) stays compatible."""
override (and the unconditional forward_impl call) stays compatible.
"""
import inspect
from vllm.v1.attention.backend import MLAAttentionImpl
@@ -177,7 +179,8 @@ _FUSED_FP8_SKIP = _fused_fp8_skip_reason()
def test_fa4_fused_fp8_output_matches_post_quant(default_vllm_config):
"""FA4's fused FP8 write (output_scale, flash-attention#135) must match the
bf16-attention + standalone static-FP8-quant path it replaces, since
production uses the same output_scale for both."""
production uses the same output_scale for both.
"""
from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
from vllm.platforms import current_platform
@@ -315,7 +315,8 @@ class TestBackendValidation:
def test_flash_attn_accepts_glm53_flash_nope_dimensions(self):
"""(256, 0, 256) runs the same FA kernels as GLM-5's (192, 64, 256); a
RoPE-carrying 320-wide query does not."""
RoPE-carrying 320-wide query does not.
"""
try:
from vllm.v1.attention.backends.mla.prefill.flash_attn import (
FlashAttnPrefillBackend,
@@ -257,7 +257,8 @@ def test_replayssm_write_pos(case: ReplaySSMBuildCase):
def test_resumed_request_differs_from_fresh():
"""Same token count, different decode_base: fresh (base 100) -> write_pos 5,
resumed (base 105) -> write_pos 0."""
resumed (base 105) -> write_pos 0.
"""
builder = _create_replayssm_builder(16)
batch = BatchSpec(seq_lens=[106, 106], query_lens=[1, 1])
common = create_common_attn_metadata(batch, BLOCK_SIZE, DEVICE).replace(
@@ -514,7 +514,6 @@ def test_mtp_builder_init_sizes_native_fp8_metadata(
Sweeping num_heads asserts metadata is sized for the padded decode shape,
covering Kimi-K3 TP4's 24 -> 32 head path and native fp8 nhead=32 folding.
"""
dtypes = SimpleNamespace(fp8="fp8", fp16="fp16", bf16="bf16")
info_calls = []
@@ -694,7 +693,6 @@ def test_full_cudagraph_padded_uniform_mtp_synthesizes_decode_indptr(
monkeypatch,
):
"""Full-CG zero-qo rows follow rocm_aiter_mla.py:608-657,717-759."""
get_mla_metadata_v1 = mock.MagicMock()
monkeypatch.setitem(
sys.modules,
@@ -756,7 +754,6 @@ def test_full_cudagraph_padded_uniform_mtp_synthesizes_decode_indptr(
def test_decode_expands_kernel_block_page_indices(monkeypatch):
"""kernel_block_size>1 expands b -> b*K+offset at rocm_aiter_mla.py:696-704."""
expand_kernel = _ExpandPageIndicesKernel()
monkeypatch.setattr(rocm_aiter_mla, "_expand_page_indices_kernel", expand_kernel)
# qlen==1 now takes the persistent-metadata path, which imports
@@ -517,7 +517,8 @@ def test_auto_selection_for_kv_connector(
aiter_found, expected_backend, mock_vllm_config, mock_get_cdna_version
):
"""Auto-selection with a KV connector and AITER enabled resolves to unified attn,
and to triton attn if AITER not enabled."""
and to triton attn if AITER not enabled.
"""
from vllm.platforms.rocm import RocmPlatform
with patch(
@@ -102,7 +102,8 @@ def expected_row_seq_lens(per_req_positions, batch_size, next_n):
def test_legacy_flat_layout_misaligns_and_bleeds():
"""The bug: ``positions[:n] + 1`` with padded-row coordinates reads other
requests' positions and prefill positions."""
requests' positions and prefill positions.
"""
decode_lens, per_req, positions = make_non_uniform_batch()
batch_size = decode_lens.shape[0]
next_n = int(decode_lens.max())
@@ -137,7 +138,8 @@ def test_helper_uniform_layout_matches_flat_slice():
def test_helper_padded_layout_per_row():
"""Non-uniform batches map every padded row to its own token's position;
pad rows get 0 (empty tail)."""
pad rows get 0 (empty tail).
"""
decode_lens, per_req, positions = make_non_uniform_batch()
out = _decode_topk_seq_lens(positions, decode_lens, 8, 3, 4, requires_padding=True)
expected = expected_row_seq_lens(per_req, 3, 4)
@@ -149,7 +151,8 @@ def test_helper_padded_layout_per_row():
def expand_tail_region(dec_seq):
"""Run the production pure-torch expand + append pair (the fused kernel
replicates it exactly on the identity path) and return the tail columns
[TOPK_TOKENS, TOPK_TOKENS + KPOOL - 1)."""
[TOPK_TOKENS, TOPK_TOKENS + KPOOL - 1).
"""
rows = dec_seq.shape[0]
pool_ids = torch.arange(SELECT_K, dtype=torch.int64).expand(rows, SELECT_K)
valid = torch.ones_like(pool_ids, dtype=torch.bool)
@@ -163,7 +166,8 @@ def expand_tail_region(dec_seq):
def test_tail_expansion_legacy_vs_fixed():
"""End-to-end tail consequence: the fixed mapping appends exactly the
request's trailing incomplete pool; the legacy flat slice drops real tail
tokens and emits indices far past the request's own sequence."""
tokens and emits indices far past the request's own sequence.
"""
decode_lens, per_req, positions = make_non_uniform_batch()
n = 12
fixed = _decode_topk_seq_lens(
+20 -11
View File
@@ -245,8 +245,8 @@ def _dequantize_fp8_ds_mla_entry(
Args:
simulate_sm100_e8m0_scales: If True, simulate the SM100 kernel's
float -> e8m0 -> bf16 scale conversion path.
"""
"""
# The first kv_lora_rank bytes store FP8 latent values with one scale per
# 128 element tile written as float32 right after the latent payload.
scales = cache_slice.view(torch.float32)[kv_lora_rank // 4 : kv_lora_rank // 4 + 4]
@@ -283,8 +283,8 @@ def _quantize_dequantize_fp8_ds_mla(
Args:
simulate_sm100_e8m0_scales: If True, simulate the SM100 kernel's
float -> e8m0 -> bf16 scale conversion in dequantization.
"""
"""
if kv_c.numel() == 0:
return kv_c.clone(), k_pe.clone()
@@ -909,7 +909,8 @@ def test_index_group_convert_during_piecewise_capture():
runs in the following eager break. The convert's side stream must not
wait on ``logical_topk_ready`` there: an event recorded inside a captured
segment is graph-local, and an eager wait on it raises
cudaErrorInvalidValue."""
cudaErrorInvalidValue.
"""
device = torch.device(DEVICE_TYPE)
num_tokens, num_topk, num_requests, blocks_per_req, block_size = 8, 128, 4, 4, 16
@@ -1048,7 +1049,8 @@ def test_triton_convert_rejects_req_id_longer_than_token_indices():
req_id but the output is allocated like token_indices, so a full-batch
req_id combined with an MQA-subset token_indices wrote past the end of
the output buffer. The wrapper must reject the length mismatch instead
of corrupting memory."""
of corrupting memory.
"""
device = torch.device(DEVICE_TYPE)
num_topk_tokens = 128
block_size = 64
@@ -1103,7 +1105,8 @@ def test_flashmla_forward_bf16_kv_slices_req_id_to_mqa_tokens():
is active, forward_mqa only receives the leading decode tokens, but
_forward_bf16_kv passed the full-batch req_id_per_token to the index
conversion, making it write past the end of its output buffer. The call
site must slice req_id_per_token to the MQA tokens."""
site must slice req_id_per_token to the MQA tokens.
"""
device = torch.device(DEVICE_TYPE)
num_topk_tokens = 128
block_size = 64
@@ -1201,7 +1204,8 @@ def test_split_prefill_chunks(seq_lens, max_buf, expected):
)
def test_masked_mha_workspace_fits_single_request_boundary(max_query_len, expected):
"""A 32K prefill needs the default workspace exactly; shrinking it would
push a supported request onto MQA."""
push a supported request onto MQA.
"""
assert (
_masked_mha_workspace_fits(
batch_size=1,
@@ -1259,7 +1263,8 @@ def test_is_masked_mha_available_model_dims(
):
"""The allow-list gates masked MHA per exact model geometry: the DeepSeek-V3.2,
GLM-5 and NoPE GLM-5.3-Flash layouts on an SM100-family GPU with FA4 and an
unquantized KV cache, nothing else."""
unquantized KV cache, nothing else.
"""
import vllm.model_executor.layers.attention.sparse_mla_attention as mod
monkeypatch.setattr(
@@ -3599,7 +3604,8 @@ def test_fp8_mixed_batch_dcp_neutralizes_empty_rows(monkeypatch):
"""A decode row whose top-k shard holds no local candidates (all -1) has
undefined kernel out/lse; it must come back as (0, -inf), the identity of
the cross-rank LSE merge, or a NaN would survive the merge even at zero
weight (0 * NaN = NaN)."""
weight (0 * NaN = NaN).
"""
num_tokens, num_heads, head_dim = 3, 2, 3
q = torch.empty(num_tokens, num_heads, head_dim, device=DEVICE_TYPE)
local_indices = torch.tensor(
@@ -3772,7 +3778,8 @@ def test_hisparse_fp8_prefill_gather_uses_dedicated_stream(monkeypatch):
def test_sparse_impl_observes_repointed_indexer_buffer():
"""The MTP proposer repoints the draft's indexer at the target model's buffer
after the backend impl is built, so the impl must resolve the buffer per read.
Snapshotting it in __init__ leaves the layer reading indices nothing writes."""
Snapshotting it in __init__ leaves the layer reading indices nothing writes.
"""
impl = object.__new__(FlashInferMLASparseImpl)
own = torch.zeros(4, 8, dtype=torch.int32)
target = torch.ones(4, 8, dtype=torch.int32)
@@ -3788,7 +3795,8 @@ def test_sparse_impl_observes_repointed_indexer_buffer():
def test_explicit_topk_buffer_supersedes_indexer():
"""Backbone skip-topk layers have no indexer, and the proposer also assigns the
shared buffer directly onto draft submodules."""
shared buffer directly onto draft submodules.
"""
impl = object.__new__(FlashInferMLASparseImpl)
indexer = SimpleNamespace(
topk_indices_buffer=torch.zeros(2, 2, dtype=torch.int32),
@@ -3803,7 +3811,8 @@ def test_explicit_topk_buffer_supersedes_indexer():
def test_sparse_mla_common_impl_resolves_buffer_lazily():
"""Guards the whole SparseMLACommonImpl family at once: a backend that
snapshots the buffer instead silently loses MTP buffer sharing."""
snapshots the buffer instead silently loses MTP buffer sharing.
"""
assert issubclass(SparseMLACommonImpl, SharedTopkIndicesBuffer)
assert isinstance(SparseMLACommonImpl.topk_indices_buffer, property), (
"topk_indices_buffer must stay a lazily-resolved property"
@@ -124,6 +124,7 @@ def _create_nvfp4_hnd_kv_cache(
Returns:
``torch.Tensor``: The nvfp4 kv_cache tensor (uint8, LBHNC-strided).
"""
# First create a bf16 cache so block tables are populated.
bf16_cache = create_and_prepopulate_kv_cache(
@@ -197,7 +198,8 @@ def _create_nvfp4_hnd_kv_cache(
def _run_trtllm_integration(batch_spec, kv_cache_dtype="auto", model_name=MODEL):
"""Run TRTLLM attention through the full FlashInfer pipeline
and compare against an SDPA reference."""
and compare against an SDPA reference.
"""
set_random_seed(42)
device = torch.device(f"{DEVICE_TYPE}:0")
@@ -425,7 +427,8 @@ def _run_trtllm_integration(batch_spec, kv_cache_dtype="auto", model_name=MODEL)
def test_trtllm_gen_full_attention_integration(batch_spec_name: str):
"""Test TRTLLM gen-full attention through the full FlashInfer
MetadataBuilder.build() -> FlashInferImpl.forward() pipeline,
with real TRTLLM kernels on Blackwell."""
with real TRTLLM kernels on Blackwell.
"""
_run_trtllm_integration(BATCH_SPECS[batch_spec_name])
@@ -436,7 +439,8 @@ def test_trtllm_gen_full_attention_integration(batch_spec_name: str):
@torch.inference_mode()
def test_trtllm_gen_nvfp4_kv_integration(batch_spec_name: str):
"""Test TRTLLM attention with nvfp4 KV cache through the full
FlashInfer MetadataBuilder.build() -> FlashInferImpl.forward() pipeline."""
FlashInfer MetadataBuilder.build() -> FlashInferImpl.forward() pipeline.
"""
_run_trtllm_integration(
BATCH_SPECS[batch_spec_name],
kv_cache_dtype="nvfp4",
-1
View File
@@ -193,7 +193,6 @@ def create_vllm_config(
hf_config_override: dict | None = None,
) -> VllmConfig:
"""Create a VllmConfig for testing with reasonable defaults."""
model_config = ModelConfig(
model=model_name,
tokenizer=model_name,
@@ -2,7 +2,8 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Fine-grained partial prefix-cache hits for hybrid (full attention + mamba
"align") models: scheduler chunk splitting, partial tail registration, CoW
on partial hits, and same-step deferral."""
on partial hits, and same-step deferral.
"""
from math import lcm
from types import SimpleNamespace
@@ -174,7 +175,8 @@ def test_mamba_align_split_partial_tail_schedule(dcp_world_size: int):
"""Chunk ends with partial hits on: block-aligned chunks, one extra stop
at the prompt's last hash boundary (registering the partial tail), then
the remaining tokens. block=512, hash=32, prompt=10000, budget=8192:
0 -> 8192 -> 9728 -> 9984 -> 10000."""
0 -> 8192 -> 9728 -> 9984 -> 10000.
"""
block_size = 512
scheduler_block_size = block_size * dcp_world_size
hash_block_size = 32
@@ -742,7 +744,8 @@ def test_hash_aligned_query_end_uses_regular_partial_tail():
def test_external_mamba_hit_same_block_uses_running_cow_on_continue():
"""An external mid-block hit must become a running request even when its
first continuation does not need another Mamba block."""
first continuation does not need another Mamba block.
"""
hash_block_size = 2
mamba_block_size = 4 * hash_block_size
kv_cache_config = KVCacheConfig(
@@ -814,7 +817,8 @@ def test_external_mamba_hit_same_block_uses_running_cow_on_continue():
def test_boundary_state_offloads_returns_cow_target():
"""Boundary hand-offs expose aligned snapshots and the partial-tail CoW
target, never the overwritten CoW source."""
target, never the overwritten CoW source.
"""
hash_block_size = 2
block_size = 2 * hash_block_size
kv_cache_config = KVCacheConfig(
@@ -1000,7 +1004,8 @@ def test_connector_finish_registers_partial_tail_before_cleanup():
def test_boundary_state_offload_dropped_when_request_freed_before_drain():
"""A hand-off recorded in the same scheduling pass as the request's death
must not be drained: its release hook has already run, so draining would
leak a pinned block."""
leak a pinned block.
"""
hash_block_size = 2
block_size = 2 * hash_block_size
kv_cache_config = KVCacheConfig(
@@ -1047,7 +1052,8 @@ def test_boundary_state_offload_dropped_when_request_freed_before_drain():
def test_boundary_state_offloads_block_aligned_prompt():
"""A prompt ending on a block boundary registers no CoW partial tail; its
boundary state block is handed off as a snapshot instead (once)."""
boundary state block is handed off as a snapshot instead (once).
"""
hash_block_size = 2
block_size = 2 * hash_block_size
kv_cache_config = KVCacheConfig(
@@ -1101,7 +1107,8 @@ def test_boundary_state_offloads_block_aligned_prompt():
def test_truncate_computed_blocks_preserves_sparse_prefix_positions():
"""truncate_computed_blocks slices each group by its own block size,
keeps null placeholders in the retained prefix, and leaves the original
lookup result untouched (pure view, no refcount changes)."""
lookup result untouched (pure view, no refcount changes).
"""
hash_block_size = 2
kv_cache_config = KVCacheConfig(
num_blocks=24,
@@ -1156,7 +1163,8 @@ def test_truncate_computed_blocks_preserves_sparse_prefix_positions():
def test_truncate_computed_blocks_allows_short_mamba_group_only():
"""External state may replace a short Mamba hit, but other groups must
cover the aligned local endpoint."""
cover the aligned local endpoint.
"""
hash_block_size = 2
kv_cache_config = KVCacheConfig(
num_blocks=24,
@@ -1298,7 +1306,8 @@ def test_hybrid_mamba_partial_tail_owner_continue_preserves_later_hit():
def test_hybrid_mamba_moved_partial_entry_defers_same_step_hit():
"""The owner's move re-arms the same-step guard: the moved entry is
filled by this step's copy, and chained same-step copies read stale
sources, so a request hitting it in the move step must be deferred."""
sources, so a request hitting it in the move step must be deferred.
"""
hash_block_size = 2
block_size = 2 * hash_block_size
kv_cache_config = KVCacheConfig(
@@ -1588,7 +1597,8 @@ def test_hybrid_partial_hash_truncates_full_attention_hit_length():
def test_cow_retained_blocks_returned_for_release():
"""new_step_starts returns the CoW copy retentions instead of freeing
them; the scheduler owns releasing them once the copy has run."""
them; the scheduler owns releasing them once the copy has run.
"""
hash_block_size = 2
block_size = 2 * hash_block_size
kv_cache_config = KVCacheConfig(
@@ -1641,7 +1651,8 @@ def test_cow_retained_blocks_returned_for_release():
def test_free_cow_retained_blocks_defers_until_copy_step_processed():
"""Scheduler releases CoW retentions immediately when the copy's step has
been processed (or deferral is off), and defers them otherwise."""
been processed (or deferral is off), and defers them otherwise.
"""
from collections import deque
freed: list = []
@@ -1678,7 +1689,8 @@ def test_free_cow_retained_blocks_defers_until_copy_step_processed():
def test_full_attention_eagle_drops_one_hash_unit():
"""With fine-grained partial hits, eagle rewinds the hit by one hash unit
instead of a whole cache block: the tail block's KV is append-only, so it
still covers the reduced length and stays in the hit as a partial block."""
still covers the reduced length and stays in the hit as a partial block.
"""
from vllm.v1.core.block_pool import BlockPool
from vllm.v1.core.single_type_kv_cache_manager import FullAttentionManager
@@ -1754,7 +1766,8 @@ def test_hybrid_partial_hit_with_eagle_stays_within_group_blocks():
"""Regression: with eagle, the mamba group must not receive the eagle
lookup margin — its finder never applies the drop, so it could return a
hit past the blocks the (dropped) full-attention group covers, crashing
the consumer's CoW with block_idx >= len(req_blocks)."""
the consumer's CoW with block_idx >= len(req_blocks).
"""
hash_block_size = 2
block_size = 2 * hash_block_size
kv_cache_config = KVCacheConfig(
@@ -2356,7 +2369,8 @@ def test_retention_snapshots_handed_off_with_exact_block_ids():
"""Under sparse retention, each retained mamba boundary state block is
handed to the connector with its exact block id. Connectors must not resolve
align-mode state blocks positionally because the block table is not
append-only."""
append-only.
"""
hash_block_size = 2
block_size = 4
manager = make_kv_cache_manager(
@@ -2397,7 +2411,8 @@ def test_snapshot_handoff_dense_default_retention():
"""With dense (default) retention, every materialized mamba boundary
state block is handed off — regular mamba-align + prefix-match-unit
deployments get store-able boundary snapshots without setting a
prefix-cache retention interval."""
prefix-cache retention interval.
"""
hash_block_size = 2
block_size = 4
manager = make_kv_cache_manager(
@@ -2429,7 +2444,8 @@ def test_snapshot_handoff_dense_default_retention():
def _run_chunked_prefill(manager, req, chunk_size, num_chunks):
"""Prefill ``req`` one ``chunk_size`` chunk per scheduler step, yielding the
boundary-state hand-offs offered (and claimed) at each step."""
boundary-state hand-offs offered (and claimed) at each step.
"""
computed_blocks, num_computed, _ = manager.get_computed_blocks(req)
for step in range(num_chunks):
manager.new_step_starts()
@@ -2493,7 +2509,8 @@ def test_boundary_states_offered_past_prompt_for_resumed_prefill():
``num_prompt_tokens``. A resumed request re-prefills its generated tokens
and every group re-saves them, so filtering on the original prompt length
would silently strip the mamba key for boundaries full attention still
stores; only the connector knows where its save window ends."""
stores; only the connector knows where its save window ends.
"""
block_size = 8
prompt_len = block_size
manager = make_kv_cache_manager(
+4 -2
View File
@@ -552,7 +552,8 @@ def _create_async_pp_scheduler(
def _assert_ordered_subset(delivered: list[int], emitted: list[int]) -> None:
"""Delivered tokens must be an order-preserving subset of the emitted
tokens with no duplicates (tokens are globally unique)."""
tokens with no duplicates (tokens are globally unique).
"""
it = iter(emitted)
for token in delivered:
assert token in it, f"token {token} delivered out of order or twice"
@@ -562,7 +563,8 @@ def _assert_positions_consistent(req, engine: PipelinedEngine) -> None:
"""The i-th delivered output token must be one the runner sampled for
exactly sequence position prompt_len + i: catches a preempted request's
stale output landing on a position the resumed request resampled (or
vice versa), which token-stream equality alone cannot see."""
vice versa), which token-stream equality alone cannot see.
"""
for i, token in enumerate(req.output_token_ids):
expected = req.num_prompt_tokens + i
actual = engine.emitted_position[token]
+12 -6
View File
@@ -178,7 +178,8 @@ def _shared_layout_config():
class TestCSALinearGrouping:
"""A CSA + linear-attention model (sparse attention with a compressor ring,
plus sharded GDN and one TP-replicated PLE state) goes through the generic
packed-group path; no model-specific branch is involved."""
packed-group path; no model-specific branch is involved.
"""
@staticmethod
def _mamba_groups(groups):
@@ -193,7 +194,8 @@ class TestCSALinearGrouping:
def test_replicated_state_gets_its_own_group(self):
"""The PLE state is TP-replicated and a different size, so it must not
share a manager group with the sharded GDN states (the NIXL worker
requires exactly one single-layer replicated group)."""
requires exactly one single-layer replicated group).
"""
groups = get_kv_cache_groups(_shared_layout_config(), _make_csa_linear_specs())
mamba_groups = self._mamba_groups(groups)
@@ -246,7 +248,8 @@ class TestCSALinearGrouping:
def test_scratch_group_survives_computed_block_truncation(self):
"""The scratch group contributes no computed blocks, so truncating a
lookup result must skip it: its block size is the ring capacity, which
neither divides the hit length nor bounds the (empty) block list."""
neither divides the hit length nor bounds the (empty) block list.
"""
config = _shared_layout_config()
config.cache_config.enable_prefix_caching = True
groups = get_kv_cache_groups(config, _make_csa_linear_specs())
@@ -360,7 +363,8 @@ class TestCSALinearGrouping:
per page size, which is emitted whole, or a single-layer state that
cannot be split at all -- the mamba layers must be sized against it.
Sizing them against a narrower bucket splits them past what the block
already fits, spending a pool block per extra group for no saving."""
already fits, spending a pool block per extra group for no saving.
"""
config = _mock_vllm_config("BLNHC")
config.speculative_config = None
specs = {}
@@ -405,7 +409,8 @@ class TestSlidingWindowBucketCap:
unbalanced 8-layer paged MLA bucket. Left whole, the SWA bucket would
set the block width and the MLA group would fill under a third of
every block; capped, it splits into groups no wider than the MLA
page."""
page.
"""
config = _shared_layout_config()
specs: dict[str, KVCacheSpec] = {}
for layer, ratio in ((2, 2), (8, 2), (14, 2), (20, 1)):
@@ -606,7 +611,8 @@ class TestCompressorRingGroup:
"""DeepSeek-V4.1 compressor state: a CircularBufferSpec ring (capacity
8) beside 128-token paged MLA groups and 64-token SWA groups. The ring
claims one block per request, never hashes, and does not disturb the
scheduler/hash block sizes the paged groups imply."""
scheduler/hash block sizes the paged groups imply.
"""
config = _shared_layout_config()
config.cache_config.enable_prefix_caching = True
specs: dict[str, KVCacheSpec] = {
+6 -3
View File
@@ -367,7 +367,8 @@ def test_free_request_with_duplicate_mm_hashes():
"""Freeing a request whose two inputs share the same mm_hash must fully
clean up request_cached_ids. After the first free_encoder_input call,
cached[mm_hash] becomes empty; the second call must still remove the
remaining input_id from request_cached_ids."""
remaining input_id from request_cached_ids.
"""
manager = EncoderCacheManager(cache_size=20)
req = MockRequest("r1", ["imgA", "imgA"], [4, 4])
@@ -383,7 +384,8 @@ def test_free_request_with_duplicate_mm_hashes():
def test_duplicate_mm_hash_stays_referenced_until_last_free():
"""`cached` holds one reference per request, not per position, so freeing
the first of two occurrences must not release the entry."""
the first of two occurrences must not release the entry.
"""
manager = EncoderCacheManager(cache_size=20)
req = MockRequest("r1", ["imgA", "imgA"], [4, 4])
@@ -404,7 +406,8 @@ def test_duplicate_mm_hash_stays_referenced_until_last_free():
def test_duplicate_mm_hash_is_not_evicted_before_last_use():
"""An item repeated within one request (an image carried across
conversation turns) must not become reclaimable capacity while a later
occurrence still has to be spliced in, or the encoder recomputes it."""
occurrence still has to be spliced in, or the encoder recomputes it.
"""
manager = EncoderCacheManager(cache_size=8)
req = MockRequest("r1", ["imgA", "imgA"], [4, 4])
other = MockRequest("r2", ["imgB"], [8])
+24 -19
View File
@@ -1069,9 +1069,7 @@ def _stats(requests: int, queries: int, hits: int) -> PrefixCacheStats:
def test_metrics():
"""
Test the prefix caching metrics.
"""
"""Test the prefix caching metrics."""
metrics = CachingMetrics(max_recent_requests=5)
assert metrics.hit_rate == 0.0
@@ -1101,9 +1099,7 @@ def test_metrics():
def test_metrics_empty_stats():
"""
Test the prefix caching metrics with empty stats.
"""
"""Test the prefix caching metrics with empty stats."""
metrics = CachingMetrics(max_recent_requests=5)
metrics.observe(_stats(0, 0, 0))
metrics.observe(_stats(1, 20, 9))
@@ -1903,7 +1899,7 @@ def test_get_max_concurrency_for_kv_cache_config():
def test_allocate_with_lookahead():
"""Verify that lookahead tokens correctly affect block allocation"""
"""Verify that lookahead tokens correctly affect block allocation."""
block_size = 4
config = KVCacheConfig(
num_blocks=10,
@@ -2597,7 +2593,8 @@ def test_get_kv_cache_config_kpool_tail_coowns_indexer_tensor():
"""The kpool tail parasitizes the indexer tensors instead of getting its
own: sibling idx/tail tensors paired by layer order, zero standalone tail
bytes, one shared block per request, and no prefix-caching leakage from
the kpool-sized scratch group."""
the kpool-sized scratch group.
"""
model_config = ModelConfig(max_model_len=8192)
vllm_config = VllmConfig(model_config=model_config)
@@ -2668,7 +2665,8 @@ def test_get_kv_cache_config_kpool_tail_coowns_indexer_tensor():
def test_glm5_kpool_tail_does_not_drag_hash_block_size():
"""The tail's kpool-sized scratch block (4 tokens) must not constrain the
prefix-cache hash granularity: participating groups alone decide it."""
prefix-cache hash granularity: participating groups alone decide it.
"""
model_config = ModelConfig(max_model_len=8192)
vllm_config = VllmConfig(model_config=model_config)
@@ -2744,7 +2742,8 @@ def test_get_kv_cache_config_mamba_hybrid_sharing_infeasible_no_indexer():
def test_get_kv_cache_config_mamba_hybrid_sharing_prepadded_mamba():
"""Platform-prepadded mamba pages (mamba_page_size_padded hint) must not
disable slot sharing; the layout re-pads them to the MLA page."""
disable slot sharing; the layout re-pads them to the MLA page.
"""
model_config = ModelConfig(max_model_len=8192)
vllm_config = VllmConfig(model_config=model_config)
@@ -2775,7 +2774,8 @@ def test_get_kv_cache_config_mamba_hybrid_sharing_prepadded_mamba():
def test_get_kv_cache_config_mamba_hybrid_sharing_pp_balanced_projection():
"""Round-robin mamba grouping keeps practical PP splits balanced: every
stage's largest projected mamba group slice fits its projected MLA
layers, so per-stage slot tensors all have an MLA owner."""
layers, so per-stage slot tensors all have an MLA owner.
"""
model_config = ModelConfig(max_model_len=8192)
vllm_config = VllmConfig(model_config=model_config)
@@ -2819,7 +2819,8 @@ def test_get_kv_cache_config_mamba_hybrid_sharing_pp_group_count_bump(monkeypatc
"""PP=4's default partition [11,11,12,11] leaves stage 0 with 2 MLA
layers but a round-robin slice of 3 under the minimum 4 mamba groups;
grouping bumps to 5 groups so every stage's projection keeps sharing
on instead of silently falling back on stage 0."""
on instead of silently falling back on stage 0.
"""
from vllm.config import ParallelConfig
from vllm.distributed.utils import get_pp_indices
@@ -2868,7 +2869,8 @@ def test_get_kv_cache_config_mamba_hybrid_sharing_pp_starved_stage(monkeypatch):
def test_get_kv_cache_config_mamba_hybrid_sharing_beats_cross_layers_flag():
"""Hybrid slot sharing must take precedence over the experimental
enable_cross_layers_blocks packed layout: generic packing would give the
MLA slots a strided view, breaking the contiguous virtual split."""
MLA slots a strided view, breaking the contiguous virtual split.
"""
model_config = ModelConfig(max_model_len=8192)
kv_transfer_config = KVTransferConfig(
kv_connector="NixlConnector",
@@ -3170,7 +3172,8 @@ def test_multi_run_layer_compact_strides_place_hoisted_heads():
"""A layer-compact run region is its own dense allocation: under LHBNC the head
groups sit between the layers and the blocks, so a run's block stride is one head
group's slice, not the whole page (regression test for setStorage out-of-bounds
on ROCm hybrid models)."""
on ROCm hybrid models).
"""
full = new_kv_cache_spec()
swa = new_sliding_window_spec(sliding_window=full.block_size * 2)
page = full.page_size_bytes
@@ -3249,7 +3252,8 @@ def test_mla_with_incompatible_swa_uses_one_full_allocation_group(caplog_vllm):
def test_hidden_states_with_tp_scales_page_size():
"""When TP shrinks KV pages below the hidden-state per-token cost,
get_kv_cache_groups must scale up target block sizes so that the
common page accommodates the unsharded hidden states."""
common page accommodates the unsharded hidden states.
"""
# Simulate TP=4 sharding a model with 8 KV heads → 2 per rank.
# KV page = block_size(16) * num_kv_heads(2) * head_size(64) * dtype(2)
# = 16 * 2 * 64 * 2 = 4096 bytes.
@@ -3922,8 +3926,7 @@ def test_unify_kv_cache_spec_page_size_mamba():
def test_hma_not_disabled_when_kv_events_enabled():
"""
Test enabling KV events must not force disable_hybrid_kv_cache_manager to True.
"""Test enabling KV events must not force disable_hybrid_kv_cache_manager to True.
This test guards against that regression by verifying that a VllmConfig
with kv_events_config set still resolves disable_hybrid_kv_cache_manager
@@ -4080,7 +4083,8 @@ def test_check_enough_kv_cache_memory_reserves_null_block():
def test_is_full_attention_spec_unwraps_uniform_type_specs():
"""``UniformTypeKVCacheSpecs`` is not a ``FullAttentionSpec``, so callers
scanning groups with a bare isinstance miss DeepSeek-V4-shaped configs."""
scanning groups with a bare isinstance miss DeepSeek-V4-shaped configs.
"""
common = dict(num_kv_heads=1, head_size=1, dtype=torch.float32)
full = FullAttentionSpec(block_size=4, **common)
mla = MLAAttentionSpec(block_size=4, **common)
@@ -4249,7 +4253,8 @@ def test_no_warning_when_draft_group_is_identified(caplog_vllm):
def _deepseek_v4_specs(model_version="deepseek_v4"):
"""DeepseekV4-shaped specs: full MLA layers plus sliding-window MLA layers
at differing page sizes, with the MTP draft layer registered last."""
at differing page sizes, with the MTP draft layer registered last.
"""
return {
"model.layers.0.self_attn.attn": new_mla_spec(),
"model.layers.1.self_attn.attn": new_mla_spec(),
+3 -6
View File
@@ -17,8 +17,7 @@ def new_kv_cache_spec():
def test_initialize_kv_cache_for_kv_sharing_different_attn_groups():
"""
Test initializing KV cache sharing with different attention groups.
"""Test initializing KV cache sharing with different attention groups.
Layers in the same KV cache group might be placed in different attn groups
if they have different attention backends.
"""
@@ -50,8 +49,7 @@ def test_initialize_kv_cache_for_kv_sharing_different_attn_groups():
def test_initialize_kv_cache_for_kv_sharing_same_attn_groups():
"""
Test case assuming that all layers in the same KV cache group have the same
"""Test case assuming that all layers in the same KV cache group have the same
attention backends. This is true for most models.
"""
shared_kv_cache_layers = {
@@ -79,8 +77,7 @@ def test_initialize_kv_cache_for_kv_sharing_same_attn_groups():
def test_initialize_kv_cache_for_kv_sharing_no_attn_groups():
"""
Test KV sharing set up when no attention groups are provided.
"""Test KV sharing set up when no attention groups are provided.
This is the case for the TPU model runner, which doesn't have
support for attention groups yet.
"""
+6 -3
View File
@@ -55,7 +55,8 @@ def _mm_feature(offset: int, length: int) -> MultiModalFeatureSpec:
def test_strip_covered_mm_data() -> None:
"""Items fully inside the computed prefix lose their data; items touching
the uncomputed region keep it; already-None data stays None."""
the uncomputed region keep it; already-None data stays None.
"""
from dataclasses import replace
covered = _mm_feature(offset=0, length=100)
@@ -107,7 +108,8 @@ def _mm_feature_mixed(offset: int, length: int) -> MultiModalFeatureSpec:
def test_strip_covered_mm_data_mrope() -> None:
"""For M-RoPE models, covered items keep their keep_on_cpu metadata fields
(the worker needs them to compute positions); payload fields are dropped."""
(the worker needs them to compute positions); payload fields are dropped.
"""
covered = _mm_feature_mixed(offset=0, length=100)
uncovered = _mm_feature_mixed(offset=300, length=100)
@@ -127,7 +129,8 @@ def test_strip_covered_mm_data_shm_address_item() -> None:
"""SHM address descriptors must survive stripping even when covered: the
worker needs the address to resolve the payload and to acknowledge the
sender's writer reference. Stripping it crashed the SHM receiver cache on
the second identical request (vllm-project/vllm#54994)."""
the second identical request (vllm-project/vllm#54994).
"""
address_item = MultiModalKwargsItem(
{
"address": MultiModalFieldElem(data=4096, field=MultiModalBatchedField()),
+64 -56
View File
@@ -126,7 +126,8 @@ def make_kv_cache_manager(kv_cache_config: KVCacheConfig, **kwargs) -> KVCacheMa
"""Build a ``KVCacheManager``, deriving ``scheduler_block_size`` from the
config (LCM of group block sizes) unless explicitly provided. This mirrors
``resolve_kv_cache_block_sizes`` for the non-context-parallel case used by
these tests, so callers don't have to pass it at every site."""
these tests, so callers don't have to pass it at every site.
"""
kwargs.setdefault(
"scheduler_block_size",
lcm(*(g.kv_cache_spec.block_size for g in kv_cache_config.kv_cache_groups)),
@@ -1826,8 +1827,7 @@ def _test_partial_request_hit(
def _make_hybrid_kv_cache_config(
block_size: int, num_blocks: int, spec_types: list[str]
) -> KVCacheConfig:
"""
Create a KVCacheConfig with the specified spec types.
"""Create a KVCacheConfig with the specified spec types.
Args:
block_size: The block size for KV cache.
@@ -1837,6 +1837,7 @@ def _make_hybrid_kv_cache_config(
- "sliding_window": SlidingWindowSpec with window=2*block_size
- "sliding_window_large": SlidingWindowSpec with window=4*block_size
- "mamba": MambaSpec
"""
spec_map = {
"full": lambda: FullAttentionSpec(
@@ -1938,8 +1939,7 @@ _HYBRID_MODEL_TEST_CASES = [
@pytest.mark.parametrize("spec_types", _HYBRID_MODEL_TEST_CASES)
def test_prefill_hybrid_model_combinations(spec_types: list[str]):
"""
Test prefix caching with hybrid models containing various combinations of
"""Test prefix caching with hybrid models containing various combinations of
KV cache spec types.
This unified test covers:
@@ -2021,8 +2021,7 @@ _EAGLE_HYBRID_MODEL_TEST_CASES = [
def test_prefill_hybrid_model_combinations_eagle(
spec_types: list[str], expect_hit_length: int
):
"""
Test prefix caching with hybrid models (1 full attn + 1 other) with EAGLE.
"""Test prefix caching with hybrid models (1 full attn + 1 other) with EAGLE.
More complex hybrid models with EAGLE are not yet supported (see issue #32802).
"""
block_size = 16
@@ -2489,8 +2488,7 @@ def test_evict():
def test_hash_block_correct_reuse():
"""
This tests when a previously cached block is reused as a new block,
"""This tests when a previously cached block is reused as a new block,
its hash metadata should be correctly reset.
"""
block_size = 16
@@ -2530,8 +2528,7 @@ def test_hash_block_correct_reuse():
def test_computed_blocks_not_evicted():
"""
Test that the computed blocks are not evicted when getting new blocks
"""Test that the computed blocks are not evicted when getting new blocks
for a request if there are any other free blocks.
"""
block_size = 16
@@ -2590,9 +2587,7 @@ def test_computed_blocks_not_evicted():
def test_basic_prefix_caching_disabled():
"""
This tests that the prefix caching is disabled.
"""
"""This tests that the prefix caching is disabled."""
block_size = 4
manager = make_kv_cache_manager(
make_kv_cache_config(block_size, 5),
@@ -2639,11 +2634,9 @@ def test_basic_prefix_caching_disabled():
@pytest.mark.parametrize("hash_fn", [sha256, sha256_cbor])
def test_cache_blocks(hash_fn):
"""
This is a unit test that tests the correctness of the _cache_full_blocks
"""This is a unit test that tests the correctness of the _cache_full_blocks
function of KVCacheManager.
"""
block_size = 4
block_pool = BlockPool(
num_gpu_blocks=5,
@@ -2688,9 +2681,7 @@ def test_cache_blocks(hash_fn):
def test_cache_blocks_multi_group():
"""
This tests that blocks are cached correctly for different kv cache groups.
"""
"""This tests that blocks are cached correctly for different kv cache groups."""
block_size = 4
block_pool = BlockPool(
num_gpu_blocks=10, enable_caching=True, hash_block_size=block_size
@@ -2773,10 +2764,7 @@ def test_cache_blocks_multi_group():
def test_mm_prefix_caching():
"""
This tests that the multi-modal prefix caching is correct.
"""
"""This tests that the multi-modal prefix caching is correct."""
block_size = 16
manager = make_kv_cache_manager(
make_kv_cache_config(block_size, 11),
@@ -2881,8 +2869,7 @@ def test_mm_prefix_caching():
def test_cache_key_salting():
"""
This tests that cache salts are applied during hashing and the cache
"""This tests that cache salts are applied during hashing and the cache
is separated cache as expected.
"""
block_size = 16
@@ -2961,8 +2948,7 @@ def test_cache_key_salting():
def test_prefill_not_enough_free_blocks_with_computed_blocks():
"""
This is a unit test that tests the correctness of the allocate_slots
"""This is a unit test that tests the correctness of the allocate_slots
when there is not enough free blocks. Specifically, when a request
has computed blocks but cannot be allocated due to not enough free blocks,
the computed blocks should not be touched.
@@ -3344,7 +3330,8 @@ def test_kv_cache_events_with_lora(blocks_to_cache: int):
@pytest.mark.parametrize("group_id", [0, 1, 2])
def test_block_stored_event_group_idx(group_id: int):
"""Test BlockStored events emitted by cache_full_blocks carry the correct
group_idx."""
group_idx.
"""
block_size = 4
num_tokens = block_size * 2
@@ -3462,8 +3449,7 @@ def test_session_id_does_not_affect_prefix_cache_identity():
def test_block_stored_event_group_idx_multiple_groups():
"""
Test BlockStored events for separate HMA groups that each carry the
"""Test BlockStored events for separate HMA groups that each carry the
correct group_idx.
Simulates the HMA scenario where full-attention blocks (group 0) and
@@ -3585,8 +3571,7 @@ def test_block_stored_event_group_idx_out_of_bounds(monkeypatch):
@pytest.mark.parametrize("group_id", [0, 1, 2])
def test_block_removed_event_group_idx(group_id: int):
"""
Test BlockRemoved events emitted on eviction carry the group_idx extracted
"""Test BlockRemoved events emitted on eviction carry the group_idx extracted
from the evicted block's BlockHashWithGroupId via get_group_id().
"""
block_size = 4
@@ -3639,7 +3624,8 @@ def test_block_removed_event_group_idx(group_id: int):
def test_emit_cached_block_events():
"""emit_cached_block_events emits one BlockStored for already-cached
(reused) prefix blocks, carrying the correct group_idx /
parent_block_hash / token_ids, and without mutating block state."""
parent_block_hash / token_ids, and without mutating block state.
"""
block_size = 4
num_cached_blocks = 3
kv_cache_group_id = 1
@@ -3752,7 +3738,8 @@ def test_emit_cached_block_events_zero_cached():
def test_eagle_enabled_removes_last_block():
"""Verify Eagle does NOT remove blocks when request
length is divisible by block size."""
length is divisible by block size.
"""
block_size = 16
manager = make_kv_cache_manager(
make_kv_cache_config(block_size, num_blocks=10),
@@ -4155,7 +4142,8 @@ def test_hybrid_cache_blocks_swa_tail_window_only():
(its right-to-left scan stops once a contiguous match is found). Blocks
earlier in the segment can never serve a hit, so
``HybridKVCacheCoordinator.cache_blocks`` should skip them rather than
polluting the prefix-cache hash map."""
polluting the prefix-cache hash map.
"""
block_size = 8
# Full attn block_size=32, SWA block_size=8, sw=8 -> lcm=32.
# tail = ceil(7/8) = 1; per_segment = 32/8 = 4.
@@ -4226,7 +4214,8 @@ def test_hybrid_cache_blocks_clamped_to_lcm():
Chunks past the last lcm-aligned boundary can never participate in a
cache hit (find_longest_cache_hit always returns lcm-aligned hits), so
caching them only pollutes the prefix-cache hash map and keeps blocks
on the LRU list that could otherwise return to the free pool."""
on the LRU list that could otherwise return to the free pool.
"""
block_size = 16
# Full attn block_size=32, SWA block_size=16 -> lcm=32.
kv_cache_config = KVCacheConfig(
@@ -4361,7 +4350,8 @@ def test_hybrid_local_kv_retention_interval_aligns_in_manager():
)
def test_hybrid_local_kv_retention_interval_rejects_invalid(interval, expected_match):
"""A retention interval that is negative or not a multiple of
scheduler_block_size errors out at construction time."""
scheduler_block_size errors out at construction time.
"""
block_size = 8
kv_cache_config = KVCacheConfig(
num_blocks=100,
@@ -4831,7 +4821,8 @@ def test_block_lookup_cache_multi_blocks_per_key():
def test_can_fit_full_sequence_swa_cap_admits_long_prompt():
"""Hybrid full+SWA model with a pool sized at the startup minimum should
admit a prompt longer than the SWA cap, because SlidingWindowManager
recycles blocks during chunked prefill (issue #39734)."""
recycles blocks during chunked prefill (issue #39734).
"""
block_size = 16
sliding_window = 4 * block_size # 64 tokens
max_num_batched_tokens = 8 * block_size # 128 tokens
@@ -4926,7 +4917,8 @@ def test_full_sequence_admission_reserves_device_blocks_for_immediate_chunk(
def test_can_fit_full_sequence_full_attention_still_gates_oversized():
"""The cap only loosens the SWA group; a prompt that exceeds the
full-attention pool capacity must still be rejected."""
full-attention pool capacity must still be rejected.
"""
block_size = 16
sliding_window = 4 * block_size
max_num_batched_tokens = 8 * block_size
@@ -5045,7 +5037,8 @@ def _take_free_blocks(manager: KVCacheManager, num_blocks: int) -> list[KVCacheB
without removing them. These ref_cnt==0 blocks stand in for evictable
cache-hit blocks left behind by a previous (e.g. preempted) request, and
sitting at the head guarantees a later group's external ``get_new_blocks``
would contend for them on unpatched code (issue #33775)."""
would contend for them on unpatched code (issue #33775).
"""
blocks: list[KVCacheBlock] = []
head = manager.block_pool.free_block_queue.fake_free_list_head
for _ in range(num_blocks):
@@ -5056,7 +5049,8 @@ def _take_free_blocks(manager: KVCacheManager, num_blocks: int) -> list[KVCacheB
def _assert_no_double_allocation(manager: KVCacheManager, req_id: str) -> None:
"""No physical block may be handed out twice across groups, and every
block referenced by the request must have a live ref_cnt."""
block referenced by the request must have a live ref_cnt.
"""
block_ids = manager.get_blocks(req_id).get_block_ids()
flat = [block_id for group in block_ids for block_id in group]
assert len(set(flat)) == len(flat), "Block IDs are not unique across groups"
@@ -5083,7 +5077,8 @@ def _cross_group_cache_hit(
) -> Request:
"""Allocate ``req_id`` with a per-group local prefix hit plus external
(connector) computed tokens, driving the coordinator's two-phase path.
Returns the allocated request so callers can free it (e.g. to preempt)."""
Returns the allocated request so callers can free it (e.g. to preempt).
"""
block_size = _two_phase_block_size(manager)
hit_blocks = _take_free_blocks(manager, num_groups * local_blocks_per_group)
cache_hit = KVCacheBlocks(
@@ -5178,7 +5173,8 @@ def test_swa_free_split_keeps_cached_tail_ahead_of_scratch():
"""Dense retention: freeing an SWA request must place its
uncached scratch blocks at the front of the free queue (recycled first)
and keep its cached checkpoint blocks at the back (retained for prefix
hits). This split is always-on, independent of the retention interval."""
hits). This split is always-on, independent of the retention interval.
"""
block_size = 8
kv_cache_config = KVCacheConfig(
num_blocks=100,
@@ -5289,7 +5285,8 @@ def _make_pure_swa_manager(block_size, sliding_window, num_blocks=100, **kwargs)
def test_pure_swa_retention_interval_caches_sparse_tails():
"""Sparse retention must work for a pure-SWA single-group model, not just
hybrid models: only the per-interval tails plus the latest replay tail are
cached, and a replay still hits the latest replayable boundary."""
cached, and a replay still hits the latest replayable boundary.
"""
block_size = 16
manager = _make_pure_swa_manager(
block_size, sliding_window=block_size, retention_interval=64
@@ -5361,7 +5358,8 @@ def test_pure_swa_retention_latest_only():
def test_pure_swa_dense_retention_caches_all():
"""With retention set to ``None``, a pure-SWA model keeps dense behavior:
every block boundary is a potential hit, so all blocks are cached."""
every block boundary is a potential hit, so all blocks are cached.
"""
block_size = 16
manager = _make_pure_swa_manager(block_size, sliding_window=block_size)
@@ -5390,7 +5388,8 @@ def test_mamba_reachable_block_mask_sparsifies_retention():
"""Mamba state-snapshot retention: with a configured retention interval,
the manager keeps one cached state per interval-sized segment (plus the
latest replay boundary) instead of a snapshot per block, which is what
lets a small attention block_size avoid Mamba dominating the KV pool."""
lets a small attention block_size avoid Mamba dominating the KV pool.
"""
from vllm.v1.core.single_type_kv_cache_manager import MambaManager
block_size = 16
@@ -5428,7 +5427,8 @@ def test_mamba_reachable_block_mask_pins_shared_prefix():
"""A Marconi-detected shared prefix (``shared_prefix_boundary``) lands before
``num_prompt`` so the replay-boundary rule alone would drop it. The mask must
pin the single state block ending on that boundary so sparse retention does
not defeat cross-request shared-prefix reuse."""
not defeat cross-request shared-prefix reuse.
"""
from vllm.v1.core.single_type_kv_cache_manager import MambaManager
block_size = 16
@@ -5476,7 +5476,8 @@ def test_mamba_shared_prefix_survives_zero_retention():
detection) keeps its Mamba state block cached under
``prefix_cache_retention_interval=0``, which otherwise retains only the
end-of-prompt replay boundary. Without this, a shared prefix (junction
before ``num_prompt``) would be recomputed by every sharing request."""
before ``num_prompt``) would be recomputed by every sharing request.
"""
block_size = 16
# 16-block (256-token) prompt; replay boundary is block 240 // 16 - 1 = 14.
@@ -5518,7 +5519,8 @@ def test_mamba_shared_prefix_reuse_under_zero_retention():
detecting request must stay reusable by a later request under
``prefix_cache_retention_interval=0``. Without the pin the junction is
masked out and the later request misses; with it (and under dense) the reuse
is preserved."""
is preserved.
"""
block_size = 16
def last_req_hit(retention, pin):
@@ -5601,7 +5603,8 @@ def test_swa_reachable_block_mask_final_partial_segment():
def test_swa_reachable_block_mask_pins_shared_prefix():
"""SWA analog of the Mamba pin: the shared-prefix junction must keep the
``need``-block sliding-window tail ending on that boundary (not a single
block), so a windowed hit can land there under sparse retention."""
block), so a windowed hit can land there under sparse retention.
"""
from vllm.v1.core.single_type_kv_cache_manager import SlidingWindowManager
block_size = 16
@@ -5646,7 +5649,8 @@ def test_swa_reachable_block_mask_pins_shared_prefix():
def test_swa_reachable_block_mask_with_dcp_scaling():
"""DCP shards each block's KV across ranks, scaling the effective block size.
Verify that dcp_world_size > 1 scales block size in reachability calculations,
producing different masks than dcp_world_size=1."""
producing different masks than dcp_world_size=1.
"""
from vllm.v1.core.single_type_kv_cache_manager import SlidingWindowManager
block_size = 16
@@ -5705,7 +5709,8 @@ def test_swa_reachable_block_mask_sub_block_alignment_is_dense(
multiple of the DCP-scaled block size) cannot be represented exactly. This
happens for hybrid offloading (e.g. Gemma), where alignment_tokens is the
full-attention chunk size and need not divide the SWA block size. The mask
must fall back to dense (None) rather than raise."""
must fall back to dense (None) rather than raise.
"""
from vllm.v1.core.single_type_kv_cache_manager import SlidingWindowManager
spec = SlidingWindowSpec(
@@ -5733,7 +5738,8 @@ def test_mamba_reachable_block_mask_ignores_dcp():
"""Mamba uses TP, not DCP: each rank holds the full recurrent state, so a
state block spans kv_cache_spec.block_size tokens regardless of DCP. The
mask must not scale by dcp_world_size, so retention granularity is
identical for any DCP world size."""
identical for any DCP world size.
"""
from vllm.v1.core.single_type_kv_cache_manager import MambaManager
spec = MambaSpec(
@@ -5767,7 +5773,8 @@ def test_mamba_reachable_block_mask_ignores_dcp():
def test_mamba_reachable_block_mask_large_dcp_stays_sparse():
"""A large DCP world size must not scale the Mamba block size, so it can
never collapse sparse retention into dense caching."""
never collapse sparse retention into dense caching.
"""
from vllm.v1.core.single_type_kv_cache_manager import MambaManager
spec = MambaSpec(
@@ -5802,7 +5809,8 @@ def test_swa_shared_prefix_reuse_under_zero_retention():
"""SWA cross-request analog: a partial shared prefix's sliding-window tail
must stay reusable under ``prefix_cache_retention_interval=0``. Without
the pin the junction window is masked out and a later request misses; with
it (and under dense) reuse is preserved."""
it (and under dense) reuse is preserved.
"""
block_size = 16
def last_req_hit(retention, pin):
+21 -21
View File
@@ -14,10 +14,10 @@ pytestmark = pytest.mark.cpu_test
class TestCheckSequenceRepetition:
"""Unit tests for the check_sequence_repetition function"""
"""Unit tests for the check_sequence_repetition function."""
def test_simple_repetition_detected(self):
"""Test detection of simple repetitive patterns"""
"""Test detection of simple repetitive patterns."""
token_ids = [1, 2, 3, 1, 2, 3, 1, 2, 3]
params = RepetitionDetectionParams(
max_pattern_size=3,
@@ -27,7 +27,7 @@ class TestCheckSequenceRepetition:
assert check_sequence_repetition(token_ids, params)
def test_repetition_below_min_count(self):
"""Test that pattern below min_count is not detected"""
"""Test that pattern below min_count is not detected."""
token_ids = [1, 2, 3, 1, 2, 3]
params = RepetitionDetectionParams(
max_pattern_size=3,
@@ -37,7 +37,7 @@ class TestCheckSequenceRepetition:
assert not check_sequence_repetition(token_ids, params)
def test_two_token_pattern(self):
"""Test detection of 2-token patterns"""
"""Test detection of 2-token patterns."""
token_ids = [1, 2, 1, 2, 1, 2, 1, 2]
params = RepetitionDetectionParams(
max_pattern_size=5,
@@ -47,7 +47,7 @@ class TestCheckSequenceRepetition:
assert check_sequence_repetition(token_ids, params)
def test_no_repetition_varied_sequence(self):
"""Test that non-repetitive sequences are not flagged"""
"""Test that non-repetitive sequences are not flagged."""
token_ids = [1, 2, 3, 4, 5, 6, 7, 8, 9]
params = RepetitionDetectionParams(
max_pattern_size=5,
@@ -57,7 +57,7 @@ class TestCheckSequenceRepetition:
assert not check_sequence_repetition(token_ids, params)
def test_partial_repetition_not_detected(self):
"""Test that incomplete repetitions are not detected"""
"""Test that incomplete repetitions are not detected."""
token_ids = [1, 2, 3, 1, 2, 3, 1, 2, 4]
params = RepetitionDetectionParams(
max_pattern_size=3,
@@ -67,7 +67,7 @@ class TestCheckSequenceRepetition:
assert not check_sequence_repetition(token_ids, params)
def test_empty_token_list(self):
"""Test with empty token list"""
"""Test with empty token list."""
params = RepetitionDetectionParams(
max_pattern_size=3,
min_pattern_size=2,
@@ -76,19 +76,19 @@ class TestCheckSequenceRepetition:
assert not check_sequence_repetition([], params)
def test_detection_disabled_max_size_zero(self):
"""Test that zero max_pattern_size disables detection"""
"""Test that zero max_pattern_size disables detection."""
token_ids = [1, 2, 1, 2, 1, 2]
params = RepetitionDetectionParams()
assert not check_sequence_repetition(token_ids, params)
def test_invalid_min_count(self):
"""Test that min_count < 2 returns False"""
"""Test that min_count < 2 returns False."""
token_ids = [1, 2, 1, 2]
params = RepetitionDetectionParams()
assert not check_sequence_repetition(token_ids, params)
def test_repetition_at_end_of_sequence(self):
"""Test detection when repetition occurs at the end"""
"""Test detection when repetition occurs at the end."""
token_ids = [1, 2, 3, 4, 5, 6, 5, 6, 5, 6]
params = RepetitionDetectionParams(
max_pattern_size=3,
@@ -98,7 +98,7 @@ class TestCheckSequenceRepetition:
assert check_sequence_repetition(token_ids, params)
def test_large_pattern_many_repetitions(self):
"""Test large pattern repeated many times"""
"""Test large pattern repeated many times."""
token_ids = [1, 2, 3, 4, 5, 6, 7, 8] * 5
params = RepetitionDetectionParams(
max_pattern_size=10,
@@ -114,10 +114,10 @@ class TestCheckSequenceRepetition:
class TestRepetitionDetectionIntegration:
"""Integration tests for repetition detection in check_stop"""
"""Integration tests for repetition detection in check_stop."""
def test_basic_repetition_stops_generation(self):
"""Test that repetition is detected and stops generation"""
"""Test that repetition is detected and stops generation."""
params = SamplingParams(
max_tokens=100,
repetition_detection=RepetitionDetectionParams(
@@ -138,7 +138,7 @@ class TestRepetitionDetectionIntegration:
assert request.stop_reason == "repetition_detected"
def test_detection_disabled_no_stop(self):
"""Test that disabled detection doesn't stop generation"""
"""Test that disabled detection doesn't stop generation."""
params = SamplingParams(
max_tokens=100,
)
@@ -152,7 +152,7 @@ class TestRepetitionDetectionIntegration:
assert not check_stop(request, max_model_len=1024)
def test_repetition_respects_min_tokens(self):
"""Test that repetition detection respects min_tokens"""
"""Test that repetition detection respects min_tokens."""
params = SamplingParams(
min_tokens=10,
max_tokens=100,
@@ -172,7 +172,7 @@ class TestRepetitionDetectionIntegration:
assert not check_stop(request, max_model_len=1024)
def test_no_repetition_continues_generation(self):
"""Test that non-repetitive tokens don't stop generation"""
"""Test that non-repetitive tokens don't stop generation."""
params = SamplingParams(
max_tokens=100,
repetition_detection=RepetitionDetectionParams(
@@ -191,7 +191,7 @@ class TestRepetitionDetectionIntegration:
assert not check_stop(request, max_model_len=1024)
def test_pattern_at_size_boundary(self):
"""Test detection at exact pattern size boundary"""
"""Test detection at exact pattern size boundary."""
params = SamplingParams(
max_tokens=100,
repetition_detection=RepetitionDetectionParams(
@@ -211,7 +211,7 @@ class TestRepetitionDetectionIntegration:
assert request.status == RequestStatus.FINISHED_REPETITION
def test_multiple_pattern_sizes_checked(self):
"""Test that function checks pattern sizes in range"""
"""Test that function checks pattern sizes in range."""
params = SamplingParams(
max_tokens=100,
repetition_detection=RepetitionDetectionParams(
@@ -231,7 +231,7 @@ class TestRepetitionDetectionIntegration:
assert request.status == RequestStatus.FINISHED_REPETITION
def test_eos_takes_precedence_over_repetition(self):
"""Test that EOS token stops before repetition check"""
"""Test that EOS token stops before repetition check."""
params = SamplingParams(
max_tokens=100,
stop_token_ids=[999],
@@ -252,7 +252,7 @@ class TestRepetitionDetectionIntegration:
assert request.status == RequestStatus.FINISHED_STOPPED
def test_min_pattern_size_filters_small_patterns(self):
"""Test that min_pattern_size filters out smaller patterns"""
"""Test that min_pattern_size filters out smaller patterns."""
params = SamplingParams(
max_tokens=100,
repetition_detection=RepetitionDetectionParams(
@@ -271,7 +271,7 @@ class TestRepetitionDetectionIntegration:
assert not check_stop(request, max_model_len=1024)
def test_high_repetition_threshold(self):
"""Test that high min_count requires many repetitions"""
"""Test that high min_count requires many repetitions."""
params = SamplingParams(
max_tokens=100,
repetition_detection=RepetitionDetectionParams(
+66 -54
View File
@@ -295,7 +295,7 @@ def test_schedule_multimodal_requests():
def test_async_scheduling_pp_allows_rescheduling_with_output_placeholders():
"""Async scheduling + PP: allow multi-step in-flight scheduling per request"""
"""Async scheduling + PP: allow multi-step in-flight scheduling per request."""
scheduler = create_scheduler(async_scheduling=True, pipeline_parallel_size=2)
(req,) = create_requests(num_requests=1, num_tokens=8)
scheduler.add_request(req)
@@ -538,7 +538,8 @@ def test_throttle_defers_inflight_prefill_chunk():
"""DP prefill balancing throttles ALL prefill compute on a throttled step,
not just new admissions: an in-progress (chunked) prefill already in the
running queue is also deferred, so the step runs decode-only, while a
separate decode keeps being scheduled."""
separate decode keeps being scheduled.
"""
scheduler = create_scheduler(
max_num_seqs=16, max_num_batched_tokens=50, enable_chunked_prefill=True
)
@@ -592,7 +593,8 @@ def test_throttle_capacity_bound_guard_admits():
"""Saturation guard: if a cadence-aligned release step cannot drain the
waiting prefill queue (it ran out of token budget), the throttle backs off on
the next step so the backlog cannot grow into a TTFT avalanche -- prefills are
admitted even though throttle_prefills is set."""
admitted even though throttle_prefills is set.
"""
scheduler = create_scheduler(
max_num_seqs=16, max_num_batched_tokens=200, enable_chunked_prefill=True
)
@@ -785,7 +787,7 @@ def test_update_from_output_routes_sampling_masks_by_request():
def test_stop_via_update_from_output():
"""Test stopping behavior through update_from_output"""
"""Test stopping behavior through update_from_output."""
scheduler = create_scheduler(num_speculative_tokens=1)
# Test case 1: Stop on EOS token
@@ -1291,7 +1293,8 @@ def test_pending_hisparse_reclamation_defers_preemption(monkeypatch):
def test_prefix_cache_query_not_inflated_by_connector_defer():
"""The GPU prefix-cache query is recorded at admission, so a request the
connector defers several times is counted once, not once per retry."""
connector defers several times is counted once, not once per retry.
"""
num_defers_before_matching = 3
scheduler = create_scheduler(
enable_prefix_caching=True,
@@ -1319,7 +1322,8 @@ def test_prefix_cache_query_not_inflated_by_connector_defer():
def test_preemption_re_records_prefix_cache_query():
"""A preempted request re-enters the lookup on resume, so its recomputation
is counted again into the preempted stats."""
is counted again into the preempted stats.
"""
scheduler = create_scheduler(enable_prefix_caching=True)
request = create_requests(num_requests=1)[0]
scheduler.add_request(request)
@@ -1355,7 +1359,8 @@ def test_preemption_re_records_prefix_cache_query():
def test_prefix_cache_stats_not_recorded_when_caching_disabled():
"""With prefix caching off there is no local lookup, so admitting a request
records no phantom miss."""
records no phantom miss.
"""
scheduler = create_scheduler(enable_prefix_caching=False)
for request in create_requests(num_requests=2):
scheduler.add_request(request)
@@ -1369,7 +1374,8 @@ def test_prefix_cache_stats_not_recorded_when_caching_disabled():
def test_prefix_cache_stats_counted_once_for_retried_then_scheduled_request():
"""A real cache hit rejected once by allocate_slots and admitted on the next
step is counted exactly once, hits included."""
step is counted exactly once, hits included.
"""
block_size = 16
scheduler = create_scheduler(
enable_prefix_caching=True,
@@ -2130,7 +2136,6 @@ def _assert_right_scheduler_output(
expected_num_scheduled_tokens: int,
):
"""Check if SchedulerOutput is correct after remote KV cache hit."""
# We should inject the kv_connector_metadata.
assert len(output.kv_connector_metadata.requests) == num_requests
@@ -2148,7 +2153,6 @@ def _assert_right_kv_cache_manager(
num_total_blocks: int,
):
"""Check whether KVCacheManager is correct after allocate."""
# Make sure the request stats are right.
EXPECTED_TOTAL_BLOCKS = num_tokens // block_size
for req in requests:
@@ -2179,7 +2183,6 @@ def _step_until_done(
model_runner_output: ModelRunnerOutput,
):
"""Loop over schedule(), update_from_output() until finished."""
all_finished = False
_ = scheduler.update_from_output(output, model_runner_output)
while not all_finished:
@@ -2207,7 +2210,6 @@ def _num_waiting_requests(scheduler: Scheduler) -> int:
def _step_until_kv_transfer_finished(scheduler: Scheduler, req_ids: list[str]):
"""Cycle requests through a KV transfer cycle."""
# Requests should first transition to WAITING_FOR_REMOTE_KVS
output = scheduler.schedule()
assert _num_waiting_requests(scheduler) == len(req_ids)
@@ -2291,11 +2293,9 @@ def test_has_sync_kv_loads(
@pytest.mark.parametrize("is_async", [False, True])
def test_kv_connector_basic(is_async: bool):
"""
Test whether Scheduler with KVConnector schedules tokens, allocates
"""Test whether Scheduler with KVConnector schedules tokens, allocates
memory, and cleans up requests as expected under normal operation.
"""
# Setup Scheduler.
BLOCK_SIZE = 16
NUM_MATCHED_NEW_TOKENS = BLOCK_SIZE * 2
@@ -2420,11 +2420,9 @@ def test_kv_connector_basic(is_async: bool):
@pytest.mark.parametrize("is_async", [False, True])
@pytest.mark.parametrize("local_cache_hits", [False, True])
def test_external_prefix_cache_metrics(is_async: bool, local_cache_hits: bool):
"""
Verify connector prefix cache metrics are updated
"""Verify connector prefix cache metrics are updated
correctly when the scheduler processes requests with KV connector hits.
"""
BLOCK_SIZE = 16
if local_cache_hits:
NUM_MATCHED_NEW_TOKENS = BLOCK_SIZE * 2 # 32 tokens
@@ -2540,11 +2538,9 @@ def test_external_prefix_cache_metrics(is_async: bool, local_cache_hits: bool):
"use_ec_connector, ec_role", [(False, None), (True, "ec_consumer")]
)
def test_kv_connector_unable_to_allocate(use_ec_connector, ec_role):
"""
Test whether scheduler with KVConnector is able to handle
"""Test whether scheduler with KVConnector is able to handle
unable to allocate (run out of blocks in allocate_slots().
"""
# Setup Scheduler With Mock External Cache Hit.
BLOCK_SIZE = 4
NUM_BLOCKS = 10
@@ -2627,11 +2623,9 @@ def test_kv_connector_unable_to_allocate(use_ec_connector, ec_role):
def test_kv_connector_handles_preemption(
is_async, use_ec_connector, ec_role, use_v2_model_runner
):
"""
Test whether scheduler with KVConnector is able to handle
"""Test whether scheduler with KVConnector is able to handle
unable to allocate (run out of blocks in allocate_slots().
"""
# Setup Scheduler With Mock External Cache Hit.
BLOCK_SIZE = 2
# NOTE: there is 1 null block, so this is 6 blocks.
@@ -2842,7 +2836,6 @@ def assert_scheduler_empty(scheduler: Scheduler):
def test_memory_leak():
"""Test that we do not have a memory leak."""
scheduler = create_scheduler(enable_prefix_caching=True)
NUM_REQUESTS = 5
@@ -2899,6 +2892,7 @@ def create_scheduler_with_priority(
Returns:
{class}`Scheduler` instance with priority scheduling
"""
model_config = ModelConfig(
model=model,
@@ -3100,7 +3094,8 @@ def create_requests_with_priority(
def test_priority_scheduling_basic_ordering():
"""Test that requests are scheduled in priority order
(lower value = higher priority)."""
(lower value = higher priority).
"""
scheduler = create_scheduler_with_priority()
# Create requests with different priorities
@@ -3129,7 +3124,8 @@ def test_priority_scheduling_basic_ordering():
def test_priority_scheduling_arrival_time_tiebreaker():
"""Test that arrival time is used
as tiebreaker when priorities are equal."""
as tiebreaker when priorities are equal.
"""
scheduler = create_scheduler_with_priority()
# Create requests with same priority but different arrival times
@@ -3286,7 +3282,8 @@ def test_priority_scheduling_preemption():
def test_priority_scheduling_no_preemption_when_space_available():
"""Test that preemption doesn't happen
when there's space for new requests."""
when there's space for new requests.
"""
scheduler = create_scheduler_with_priority(
max_num_seqs=3, # Allow 3 concurrent requests
max_num_batched_tokens=200, # Sufficient token budget
@@ -3339,7 +3336,8 @@ def test_priority_scheduling_no_preemption_when_space_available():
def test_priority_scheduling_preemption_victim_selection():
"""Test that the correct victim is selected for
preemption based on priority and arrival time."""
preemption based on priority and arrival time.
"""
# This test verifies the priority-based victim selection logic
# by checking the waiting queue order after adding requests with different
# priorities
@@ -3459,7 +3457,8 @@ def test_priority_scheduling_waiting_queue_order():
def test_priority_scheduling_fcfs_fallback():
"""Test that FCFS behavior is maintained when all
requests have same priority."""
requests have same priority.
"""
scheduler = create_scheduler_with_priority()
# Create requests with same priority but different arrival times
@@ -3528,7 +3527,8 @@ def test_priority_scheduling_with_limited_slots():
def test_priority_scheduling_heap_property():
"""Test that the waiting queue maintains heap
property for priority scheduling."""
property for priority scheduling.
"""
scheduler = create_scheduler_with_priority(
max_num_seqs=1, # Only one request can run at a time
)
@@ -3753,7 +3753,8 @@ def test_priority_scheduling_preemption_and_resumption_when_out_of_kv(
use_ec_connector, ec_role, use_v2_model_runner
):
"""Test that priority scheduling preempts lower priority requests
when out of KV cache space."""
when out of KV cache space.
"""
# Create scheduler with very limited memory to force preemption
scheduler = create_scheduler_with_priority(
max_num_seqs=2, # Allow multiple requests
@@ -3898,7 +3899,8 @@ def test_chunked_prefill_disabled_for_encoder_decoder(
enable_chunked_prefill: bool, is_encoder_decoder: bool, expect_enabled: bool
) -> None:
"""Validate that chunked prefill is appropriately disabled for
encoder-decoder models."""
encoder-decoder models.
"""
scheduler_config = SchedulerConfig(
enable_chunked_prefill=enable_chunked_prefill,
is_encoder_decoder=is_encoder_decoder,
@@ -3927,7 +3929,8 @@ def _validate_chunked_prefill_settings_for_encoder_decoder(
scheduler_config: SchedulerConfig, is_encoder_decoder: bool, expect_enabled: bool
) -> None:
"""Validate chunked prefill settings in the scheduler config for
encoder-decoder models."""
encoder-decoder models.
"""
assert scheduler_config.enable_chunked_prefill is expect_enabled
if is_encoder_decoder:
# Encoder-decoder models should automatically disable chunked multimodal
@@ -3987,7 +3990,7 @@ def _assert_right_ec_connector_metadata(
output: SchedulerOutput,
mm_features_list: list[MultiModalFeatureSpec],
):
"""Verify that ECConnector metadata EXACTLY matches the input MM data"""
"""Verify that ECConnector metadata EXACTLY matches the input MM data."""
# Get the connector metadata
metadata = output.ec_connector_metadata
@@ -4017,8 +4020,8 @@ def _assert_right_encoder_inputs(
):
"""Verify that requests/mm_hashes should (not) in scheduled encoder input
If check_exist is False, this function returns True
if requests are NOT in encoder inputs"""
if requests are NOT in encoder inputs
"""
# Get the scheduled encoder inputs
# NOTE: scheduled_encoder_inputs is a dictionary with request id as key
scheduled_encoder_inputs = output.scheduled_encoder_inputs
@@ -4179,7 +4182,8 @@ def test_ec_connector_text_only_request(use_kv_connector):
@pytest.mark.parametrize("use_kv_connector", [False, True])
def test_ec_connector_cache_hit_external_load(use_kv_connector):
"""Test ec_consumer loads from external cache when hit.
A normal basic operation for EPD disaggrgation"""
A normal basic operation for EPD disaggrgation
"""
scheduler = create_scheduler(
model="llava-hf/llava-1.5-7b-hf",
enable_prefix_caching=True,
@@ -4527,11 +4531,9 @@ def test_ec_connector_schedule_multiple_requests(cache_exist, use_kv_connector):
@pytest.mark.parametrize("use_kv_connector", [False, True])
def test_ec_connector_unable_to_allocate(use_kv_connector):
"""
Test whether scheduler with ECConnector is able to handle
"""Test whether scheduler with ECConnector is able to handle
unable to allocate (run out of blocks).
"""
# Setup Scheduler With Mock External Cache Hit.
BLOCK_SIZE = 4
NUM_BLOCKS = 10
@@ -4628,7 +4630,8 @@ def test_priority_scheduling_ec_connector_preemption_and_resumption(
cache_exist, use_kv_connector
):
"""Test that priority scheduling preempts lower priority requests
when out of KV cache space."""
when out of KV cache space.
"""
# Create scheduler with very limited memory to force preemption
scheduler = create_scheduler_with_priority(
model="llava-hf/llava-1.5-7b-hf",
@@ -4860,8 +4863,7 @@ def test_priority_scheduling_ec_connector_preemption_and_resumption(
@pytest.mark.parametrize("use_kv_connector", [False, True])
def test_ec_connector_allocate_encoder_tokens_with_external_load(use_kv_connector):
"""
Scenario:
"""Scenario:
- Encoder cache size: 32
- Request A: 1 feature (12 tokens) → NOT cached remotely.
- Request B: 3 features (3 x 10 tokens) → ALL cached remotely.
@@ -5298,7 +5300,8 @@ def test_scheduler_kv_connector_stats():
def test_ec_connector_update_connector_output_called():
"""Test that worker-side EC connector output is forwarded to the
EC connector's update_connector_output hook."""
EC connector's update_connector_output hook.
"""
scheduler = create_scheduler(
model="llava-hf/llava-1.5-7b-hf",
use_ec_connector=True,
@@ -5503,7 +5506,8 @@ def test_variable_length_cross_attn_block_allocation():
def test_cross_attn_blocks_not_over_allocated():
"""Test that cross-attention blocks are not over-allocated compared to
what each request actually needs."""
what each request actually needs.
"""
from math import ceil
block_size = 16
@@ -5599,7 +5603,8 @@ def test_cross_attn_blocks_not_under_allocated():
def test_cross_attn_zero_blocks_without_encoder_inputs():
"""Test that requests without encoder inputs get zero cross-attention
blocks, even when the scheduler is configured for encoder-decoder."""
blocks, even when the scheduler is configured for encoder-decoder.
"""
block_size = 16
scheduler = _create_encoder_decoder_scheduler(block_size=block_size)
@@ -5704,7 +5709,8 @@ def test_free_encoder_inputs_respects_unconfirmed_placeholders():
evicted entry and crashes the engine with "Encoder cache miss". The
scheduler must retain the input until the *confirmed* progress
(num_computed_tokens - num_output_placeholders) passes the range end, so
that no pending rejection can rewind into the range."""
that no pending rejection can rewind into the range.
"""
scheduler = create_scheduler(
model="llava-hf/llava-1.5-7b-hf",
num_speculative_tokens=3,
@@ -5815,7 +5821,8 @@ def test_free_encoder_inputs_defers_for_eagle_lookahead():
position so the drafter's +1 look-ahead mm-embedding gather (which reads one
position past the target's computed range) still finds it cached. This is
the primary mechanism that prevents the drafter "Encoder cache miss"; the
worker-side token-embedding fallback is only a backstop."""
worker-side token-embedding fallback is only a backstop.
"""
scheduler = create_scheduler(model="llava-hf/llava-1.5-7b-hf")
# create_scheduler only builds ngram spec configs; force the eagle path that
# _free_encoder_inputs keys off (its read-ahead deferral).
@@ -5845,7 +5852,8 @@ def test_free_encoder_inputs_defers_for_eagle_lookahead():
def test_free_encoder_inputs_unchanged_without_spec_decode():
"""Without speculative decoding, encoder inputs are freed as soon as
num_computed_tokens passes the placeholder range, as before."""
num_computed_tokens passes the placeholder range, as before.
"""
scheduler = create_scheduler(model="llava-hf/llava-1.5-7b-hf")
mm_positions = [[PlaceholderRange(offset=50, length=100)]]
request = create_requests(
@@ -5876,7 +5884,8 @@ def test_encoder_cache_retained_across_preemption_and_resume():
encoder input must pull the still-cached entry back out of `freeable`
without scheduling a recompute, keeping the scheduler and worker
consistent. The spec-rollback retention margin does not gate this path,
so it is covered separately here."""
so it is covered separately here.
"""
scheduler = create_scheduler(
model="llava-hf/llava-1.5-7b-hf",
num_speculative_tokens=3,
@@ -5920,7 +5929,8 @@ def test_encoder_cache_recomputed_when_evicted_during_preemption():
pressure before it resumes, the scheduler reports the mm_hash as freed
(so the worker drops it) and a resume must schedule a recompute rather
than assume the worker still holds it. check_and_update_cache must
return False so the encoder input is re-scheduled."""
return False so the encoder input is re-scheduled.
"""
scheduler = create_scheduler(
model="llava-hf/llava-1.5-7b-hf",
num_speculative_tokens=3,
@@ -6571,7 +6581,8 @@ def _decode_ready_request(scheduler):
def test_update_draft_token_ids_strips_ngram_padding():
"""ngram_gpu pads unfilled draft slots with -1; manager validation must
strip them before grammar.validate_tokens (which otherwise raises)."""
strip them before grammar.validate_tokens (which otherwise raises).
"""
scheduler = create_scheduler(num_speculative_tokens=4)
request = _decode_ready_request(scheduler)
@@ -6590,7 +6601,8 @@ def test_update_draft_token_ids_strips_ngram_padding():
def test_update_draft_token_ids_in_output_strips_padding():
"""Same guard on the output path; the -1 pad-back for the rejected count
is preserved (only the input to manager validation is stripped)."""
is preserved (only the input to manager validation is stripped).
"""
scheduler = create_scheduler(num_speculative_tokens=4)
request = _decode_ready_request(scheduler)
@@ -97,7 +97,8 @@ def test_num_in_flight_tokens_accounting():
def test_swa_free_waits_for_in_flight_step():
"""Async: out-of-window blocks stay allocated until the step that still
reads them has been processed."""
reads them has been processed.
"""
scheduler = _create_swa_scheduler(async_scheduling=True)
request = create_requests(
num_requests=1, num_tokens=NUM_PROMPT_TOKENS, block_size=BLOCK_SIZE
@@ -132,7 +133,8 @@ def test_swa_free_waits_for_in_flight_step():
def test_swa_free_immediate_when_sync():
"""Sync: no in-flight step at schedule time, frees happen at the first
decode allocation as before."""
decode allocation as before.
"""
scheduler = _create_swa_scheduler(async_scheduling=False)
request = create_requests(
num_requests=1, num_tokens=NUM_PROMPT_TOKENS, block_size=BLOCK_SIZE
@@ -171,7 +173,8 @@ def test_swa_admission_cap_accounts_for_overlapping_batches():
def test_chunked_local_free_waits_for_in_flight_step():
"""Chunked-local attention frees whole chunks left of the current one, and
is exposed to the same load-WAR: with async scheduling those chunks must
stay allocated until the in-flight step that still reads them settles."""
stay allocated until the in-flight step that still reads them settles.
"""
scheduler = _create_chunked_scheduler(async_scheduling=True)
request = create_requests(
num_requests=1, num_tokens=NUM_PROMPT_TOKENS, block_size=BLOCK_SIZE
@@ -226,7 +229,8 @@ def test_connector_finish_frees_on_settled_basis():
is handed to a KV connector (simple CPU offload / NIXL store), must use the
same processed-token basis. Otherwise the connector reads/hands off a block
the still-in-flight step is optimistically counted as done with, which is
the load-path WAR this fix closes."""
the load-path WAR this fix closes.
"""
scheduler = create_scheduler(
block_size=BLOCK_SIZE,
async_scheduling=True,
+1
View File
@@ -89,6 +89,7 @@ def create_scheduler(
Returns:
{class}`Scheduler` instance
"""
model_config = ModelConfig(
model=model,
@@ -1,8 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Unit tests for the breakable cudagraph primitives.
"""
"""Unit tests for the breakable cudagraph primitives."""
from __future__ import annotations
@@ -82,7 +80,8 @@ def test_piecewise_capture_builds_fresh_metadata_for_both_passes():
@pytest.fixture(autouse=True)
def _reset_breakable_tls():
"""Defensively clear thread-local capture state between tests so a
failure in one test can't leak "nested capture" errors into the next."""
failure in one test can't leak "nested capture" errors into the next.
"""
from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphCapture
BreakableCUDAGraphCapture._tls.active = None
@@ -292,7 +291,8 @@ def test_capture_replay_matches_eager_simple(cuda_capture_stream):
def test_decorator_breaks_when_invoked_inside_capture(cuda_capture_stream):
"""Verify @eager_break_during_capture correctly routes through
add_eager when inside a capture context, and runs straight through
when there's no active capture."""
when there's no active capture.
"""
from vllm.compilation.breakable_cudagraph import (
BreakableCUDAGraphCapture,
eager_break_during_capture,
@@ -554,7 +554,8 @@ def _mock_vllm_config():
def test_wrapper_falls_through_on_runtime_mode_mismatch():
"""A PIECEWISE-scoped wrapper runs other modes eagerly, capturing
nothing, so an outer CUDAGraphWrapper can own them."""
nothing, so an outer CUDAGraphWrapper can own them.
"""
from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphWrapper
from vllm.config import CUDAGraphMode
from vllm.forward_context import BatchDescriptor, set_forward_context
@@ -376,7 +376,6 @@ def _run_and_monitor_call(
wrapper, input_tensor, runtime_mode, batch_descriptor, vllm_config
):
"""Helper to run a single call and monitor the action."""
with (
patch("torch.cuda.graph", wraps=torch.cuda.graph) as mock_graph_context,
patch.object(wrapper, "runnable", wraps=wrapper.runnable) as mock_runnable,
+6 -10
View File
@@ -31,8 +31,7 @@ def test_v1_generation_is_deterministic_across_batch_sizes_with_needle(
backend,
rms_norm_impl,
):
"""
Ensures that the same request (the 'needle' prompt) yields identical output
"""Ensures that the same request (the 'needle' prompt) yields identical output
whether run alone (bs=1) or mixed into a larger batch (e.g., bs=64),
using the high-level v1 LLM() API only (no manual batching).
@@ -51,6 +50,7 @@ def test_v1_generation_is_deterministic_across_batch_sizes_with_needle(
to produce a more random-sounding phrase, yet remain deterministic by
seed.
- Keep max_tokens and max_model_len bounded for speed and memory use.
"""
seed = int(os.getenv("VLLM_TEST_SEED", "12345"))
random.seed(seed)
@@ -395,8 +395,7 @@ def test_logprobs_bitwise_batch_invariance_bs1_vs_bsN(
BACKENDS,
)
def test_simple_generation(backend):
"""
Simple test that runs the model with a basic prompt and prints the output.
"""Simple test that runs the model with a basic prompt and prints the output.
Useful for quick smoke testing and debugging.
"""
model = TEST_MODEL
@@ -447,8 +446,7 @@ def test_simple_generation(backend):
def test_logprobs_without_batch_invariance_should_fail(
backend, monkeypatch: pytest.MonkeyPatch
):
"""
This test is the inverse of test_logprobs_bitwise_batch_invariance_bs1_vs_bsN.
"""This test is the inverse of test_logprobs_bitwise_batch_invariance_bs1_vs_bsN.
It DISABLES batch invariance mode and expects to see non-deterministic behavior
between BS=1 and BS=N runs. This demonstrates that batch invariance is actually
doing something useful.
@@ -660,8 +658,7 @@ def test_logprobs_without_batch_invariance_should_fail(
def test_decode_logprobs_match_prefill_logprobs(
backend,
):
"""
Test that verifies decode logprobs match prefill logprobs.
"""Test that verifies decode logprobs match prefill logprobs.
For each decoded token at position i:
1. Run decode to generate N tokens and collect their logprobs
@@ -927,8 +924,7 @@ def LLM_with_max_seqs(
attention_config: dict | None = None,
kernel_config: dict | None = None,
) -> LLM:
"""
Helper to construct an LLM with a specific max_num_seqs (batch-size limit)
"""Helper to construct an LLM with a specific max_num_seqs (batch-size limit)
using the high-level v1 LLM API, while constraining memory usage.
"""
extra_kwargs: dict = {}
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Test batch-invariant matmul against torch.matmul for various shape combinations.
"""Test batch-invariant matmul against torch.matmul for various shape combinations.
Tests correctness (matches torch.matmul) and batch invariance (result for one
item doesn't change based on other items in the batch).
@@ -53,9 +52,7 @@ DEVICE_TYPE = current_platform.device_type
)
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
def test_matmul_correctness(a_shape, b_shape, dtype):
"""
Compare matmul_batch_invariant against torch.matmul for various shapes.
"""
"""Compare matmul_batch_invariant against torch.matmul for various shapes."""
device = torch.device(DEVICE_TYPE)
torch.manual_seed(42)
@@ -87,11 +84,9 @@ def test_matmul_correctness(a_shape, b_shape, dtype):
@skip_unsupported
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
def test_matmul_batch_invariance(dtype):
"""
Verify that the result for one item is bitwise identical regardless
"""Verify that the result for one item is bitwise identical regardless
of what other items are in the batch.
"""
device = torch.device(DEVICE_TYPE)
torch.manual_seed(42)
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
HTTP-based batch invariance test: send requests to a running
"""HTTP-based batch invariance test: send requests to a running
vLLM server and compare BS=1 vs BS=N results (tokens and per-step logprobs).
Environment variables:
@@ -41,8 +41,7 @@ def test_rms_norm_batch_invariant_vs_reference(
eps: float,
seed: int,
):
"""
Compare batch-invariant Triton RMS norm against a PyTorch reference.
"""Compare batch-invariant Triton RMS norm against a PyTorch reference.
Tests that the Triton-based batch-invariant RMS norm produces numerically
equivalent results to an independent implementation across configurations.
@@ -90,9 +89,7 @@ def test_fused_add_rms_norm_batch_invariant_residual_path(
n_extra: int,
seed: int,
):
"""
Test the batch-invariant fused residual-add + RMSNorm helper directly.
"""
"""Test the batch-invariant fused residual-add + RMSNorm helper directly."""
device = torch.device(DEVICE_TYPE)
torch.manual_seed(seed)
@@ -278,7 +275,8 @@ def test_rms_norm_per_block_quant_batch_invariant(
hidden_size: int, dtype: torch.dtype, seed: int
):
"""C++ ``rms_norm_per_block_quant`` must be batch invariant across the
block threshold (compiled fp8 block-quant path; block pinned to 512)."""
block threshold (compiled fp8 block-quant path; block pinned to 512).
"""
import vllm._custom_ops as ops
device = torch.device(DEVICE_TYPE)
@@ -313,8 +311,7 @@ def test_rms_norm_per_block_quant_batch_invariant(
def test_rms_norm_3d_input(
default_vllm_config, batch_size: int, seq_len: int, hidden_size: int
):
"""
Test RMS norm with 3D input tensors (batch, seq_len, hidden_size).
"""Test RMS norm with 3D input tensors (batch, seq_len, hidden_size).
Ensures that the batch-invariant RMS norm correctly handles multi-dimensional
inputs that are common in transformer models.
@@ -349,8 +346,7 @@ def test_rms_norm_3d_input(
@skip_if_not_cuda
def test_rms_norm_numerical_stability(default_vllm_config):
"""
Test RMS norm numerical stability with extreme values.
"""Test RMS norm numerical stability with extreme values.
Ensures that both implementations handle edge cases like very small or large
values without producing NaN or Inf.
@@ -406,8 +402,7 @@ def test_rms_norm_numerical_stability(default_vllm_config):
@skip_unsupported
def test_rms_norm_formula(default_vllm_config):
"""
Test that RMS norm follows the correct mathematical formula.
"""Test that RMS norm follows the correct mathematical formula.
Verifies: output = input / sqrt(mean(input^2) + eps) * weight
"""
@@ -440,8 +435,7 @@ def test_rms_norm_formula(default_vllm_config):
@skip_if_not_cuda
@pytest.mark.parametrize("hidden_size", [128, 1024, 4096, 16384])
def test_rms_norm_different_hidden_sizes(default_vllm_config, hidden_size: int):
"""
Test RMS norm with various hidden sizes to ensure block size handling.
"""Test RMS norm with various hidden sizes to ensure block size handling.
The Triton kernel uses a fixed BLOCK_SIZE=1024, so this tests that it
correctly handles hidden sizes both smaller and larger than the block size.
@@ -474,8 +468,7 @@ def test_rms_norm_different_hidden_sizes(default_vllm_config, hidden_size: int):
@skip_unsupported
def test_rms_norm_determinism(default_vllm_config):
"""
Test that batch-invariant RMS norm produces deterministic results.
"""Test that batch-invariant RMS norm produces deterministic results.
Runs the same input through the kernel multiple times and verifies
identical outputs.
+9 -7
View File
@@ -493,7 +493,6 @@ async def test_dp_pause_abort(expert_parallel: bool):
@pytest.mark.parametrize("expert_parallel", [False, True])
async def test_dp_pause_keep_then_resume(expert_parallel: bool):
"""Start generation, pause after a few tokens (keep mode), resume; verify gap."""
pause_duration = 2.0
min_tokens_before_pause = 3
@@ -548,7 +547,8 @@ async def test_dp_pause_keep_then_resume(expert_parallel: bool):
async def test_dp_pause_keep_race_staggered_engines():
"""Race: send pause(keep) to engine 0, then add two requests,
then pause(keep) to engine 1. Ensures no deadlock when pause
requests are staggered and requests arrive in between."""
requests are staggered and requests arrive in between.
"""
if DP_SIZE != 2:
pytest.skip("test_dp_pause_keep_race_staggered_engines requires DP_SIZE=2")
@@ -607,8 +607,7 @@ async def test_dp_pause_keep_race_staggered_engines():
@pytest.mark.asyncio
async def test_dp_pause_barrier_request_deadlock():
"""
Test that start_dp_wave is ignored while paused.
"""Test that start_dp_wave is ignored while paused.
Sequence:
1. Pause all engines (PAUSED_ALL).
@@ -719,7 +718,8 @@ async def test_dp_pause_barrier_request_deadlock():
async def test_dp_pause_wait_mode_drains_in_flight():
"""mode="wait" through the DP consensus path: the pause lets the
in-flight request run to completion while the consensus is pending,
resolves only once drained, and the engine generates again on resume."""
resolves only once drained, and the engine generates again on resume.
"""
with ExitStack() as after:
engine = AsyncLLM.from_engine_args(_get_dp_pause_engine_args(True))
after.callback(engine.shutdown)
@@ -750,7 +750,8 @@ async def test_dp_pause_wait_mode_drains_in_flight():
async def test_dp_pause_while_asleep():
"""Pausing a sleeping DP engine: sleeping ranks skip dummy batches yet
must still reach the pause consensus, and the completion barrier must
be harmless on workers whose memory is unmapped."""
be harmless on workers whose memory is unmapped.
"""
with ExitStack() as after:
engine = AsyncLLM.from_engine_args(_get_dp_pause_engine_args(True))
after.callback(engine.shutdown)
@@ -786,7 +787,8 @@ async def test_dp_pause_completion_implies_device_idle(
"""A resolved pause future promises an idle device. Inflate the dummy
batches the pause consensus manufactures so any work the pause fails to
wait on stays visible, then require a quiet worker stream the moment
pause_generation() returns."""
pause_generation() returns.
"""
with ExitStack() as after:
# The probes below ship functions through collective_rpc.
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
+2 -5
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Test Dual Batch Overlap (DBO) with Data Parallelism + Expert Parallelism.
"""Test Dual Batch Overlap (DBO) with Data Parallelism + Expert Parallelism.
DBO is specifically designed for DP+EP scenarios to hide communication latency
by overlapping computation of two batches. This test validates that DBO works
@@ -54,9 +53,7 @@ DEEPEP_BACKENDS = [
),
)
def test_dbo_dp_ep_gsm8k(all2all_backend: str, num_gpus_available):
"""
Test DBO with DP+EP using GSM8K evaluation.
"""
"""Test DBO with DP+EP using GSM8K evaluation."""
lm_eval = pytest.importorskip("lm_eval")
required_gpus = DP_SIZE
+2 -1
View File
@@ -24,7 +24,8 @@ TP_SIZE = int(os.getenv("TP_SIZE", "1"))
class ExternalLBServerManager:
"""Manages data parallel vLLM server instances for external
load balancer testing."""
load balancer testing.
"""
def __init__(
self,
+2 -1
View File
@@ -30,7 +30,8 @@ DP_SIZE_LOCAL = DP_SIZE // NUM_NODES # 2 ranks per node
class HybridLBServerManager:
"""Manages hybrid data parallel vLLM server instances where each node
runs a single logical API server that balances requests only to the
DP engines running on that same node."""
DP engines running on that same node.
"""
def __init__(
self,
+6 -5
View File
@@ -107,7 +107,8 @@ async def _run_request_bursts(
class MultinodeInternalLBServerManager:
"""Manages multi-node data parallel vLLM server instances for internal
load balancer testing using --headless mode."""
load balancer testing using --headless mode.
"""
def __init__(
self,
@@ -241,7 +242,8 @@ class MultinodeInternalLBServerManager:
class APIOnlyServerManager:
"""Manages API-only server (Node 0) and headless engines server (Node 1)
for testing separated API server and engine configuration."""
for testing separated API server and engine configuration.
"""
def __init__(
self,
@@ -261,7 +263,6 @@ class APIOnlyServerManager:
def __enter__(self) -> list[tuple[RemoteOpenAIServer, list[str]]]:
"""Start API-only server and headless engines server."""
# Start API-only server (Node 0) - no engines, only API server
api_server_args = self.base_server_args.copy()
api_server_args.extend(
@@ -607,7 +608,6 @@ async def test_api_only_multinode_dp_completion(
model_name: str,
) -> None:
"""Test API-only server with all engines on separate headless server."""
# Test single request
result = await _make_completion_request(api_only_client, model_name)
assert result is not None
@@ -645,7 +645,8 @@ async def test_api_only_multinode_dp_completion_streaming(
model_name: str,
) -> None:
"""Test API-only server streaming with all engines on separate
headless server."""
headless server.
"""
prompt = "What is an LLM?"
async def make_streaming_request():
+8 -6
View File
@@ -82,7 +82,8 @@ async def _generate(engine: AsyncLLM, prompt: str, max_tokens: int) -> int:
@pytest.mark.parametrize("async_scheduling", [True, False])
async def test_pp_dp_v2_low_concurrency(async_scheduling: bool):
"""A single in-flight request at a time, repeated, to exercise the
PP slot ring under empty batches between decodes."""
PP slot ring under empty batches between decodes.
"""
with ExitStack() as after:
engine = AsyncLLM.from_engine_args(_engine_args(async_scheduling))
after.callback(engine.shutdown)
@@ -96,7 +97,8 @@ async def test_pp_dp_v2_low_concurrency(async_scheduling: bool):
@pytest.mark.parametrize("async_scheduling", [True, False])
async def test_pp_dp_v2_mid_concurrency(async_scheduling: bool):
"""64 concurrent requests, staggered, to exercise the steady-state
DP all-reduce + PP slot-ring path."""
DP all-reduce + PP slot-ring path.
"""
with ExitStack() as after:
engine = AsyncLLM.from_engine_args(_engine_args(async_scheduling))
after.callback(engine.shutdown)
@@ -112,8 +114,8 @@ async def test_pp_dp_v2_mid_concurrency(async_scheduling: bool):
@pytest.mark.asyncio
async def test_pp_dp_v2_abort_mid_decode():
"""Cancel half the in-flight requests mid-stream and confirm the
engine survives the abort storm."""
engine survives the abort storm.
"""
with ExitStack() as after:
engine = AsyncLLM.from_engine_args(_engine_args(async_scheduling=True))
after.callback(engine.shutdown)
@@ -153,8 +155,8 @@ async def test_pp_dp_v2_abort_mid_decode():
@pytest.mark.asyncio
async def test_pp_dp_v2_pause_resume():
"""Pause an engine with a request in flight, then resume and confirm
new requests still work."""
new requests still work.
"""
with ExitStack() as after:
engine = AsyncLLM.from_engine_args(_engine_args(async_scheduling=True))
after.callback(engine.shutdown)
@@ -48,7 +48,8 @@ def test_without_spec_decoding(
monkeypatch: pytest.MonkeyPatch,
):
"""Test consistency of combos of async scheduling, preemption,
uni/multiproc executor, prefill chunking."""
uni/multiproc executor, prefill chunking.
"""
struct_outputs = StructuredOutputsParams(json=sample_json_schema)
test_sampling_params: list[dict[str, Any]] = [
dict(),
@@ -111,7 +112,6 @@ def test_with_eagle3_spec_decoding(sample_json_schema, monkeypatch: pytest.Monke
preemption, executor, async scheduling, prefill chunking,
spec decoding model length.
"""
spec_config = {
"method": "eagle3",
"num_speculative_tokens": 2,
@@ -171,7 +171,6 @@ def test_with_ngram_gpu_spec_decoding(monkeypatch: pytest.MonkeyPatch):
- Async scheduling enabled (as in production)
- Different executors and chunking settings
"""
# Variant with larger speculation window
ngram_gpu_config = {
"method": "ngram_gpu",
@@ -206,8 +205,8 @@ def run_tests(
test_sampling_params: list[dict[str, Any]],
):
"""Test consistency of combos of async scheduling, preemption,
uni/multiproc executor with spec decoding."""
uni/multiproc executor with spec decoding.
"""
# Flex attention supports float32.
attention_config = {"backend": "FLEX_ATTENTION"}
@@ -72,7 +72,8 @@ def test_backend_per_kind_splits_groups(backend_per_kind, monkeypatch):
)
def test_backend_per_kind_overrides_global_backend(monkeypatch):
"""A per-kind entry wins over the global ``backend`` for its kind; other
kinds fall back to the global backend."""
kinds fall back to the global backend.
"""
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
llm = LLM(
model=MODEL,
+5 -6
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Tests for vLLM `vllm/v1/engine/processor.Processor._validate_model_input()`
"""Tests for vLLM `vllm/v1/engine/processor.Processor._validate_model_input()`
handling of maximum context length for decoder models.
This test ensures:
@@ -35,8 +34,8 @@ def test_decoder_max_context_length_validation(
max_tokens: int,
) -> None:
"""Check vLLM decoder model input validation for edge cases where
the prompt length is (almost) equal to the max model length."""
the prompt length is (almost) equal to the max model length.
"""
prompt_ids = [[43] * prompt_len]
with vllm_runner(
@@ -73,8 +72,8 @@ def test_auto_fit_max_model_len_rejects_oversized_input(
"""When max_model_len='auto' and KV cache memory is very limited,
the engine auto-fits max_model_len to a small value. The frontend
must see this reduced value and reject prompts that exceed it,
rather than accepting them and hanging."""
rather than accepting them and hanging.
"""
# Use a small KV cache budget to force auto-fit to a small
# max_model_len. Pin block_size=16 so the budget is independent
# of the platform's default block size. One block for this model is
@@ -35,8 +35,7 @@ model_config = {
def test_sliding_window_retrieval(
model, batch_size, seed, disable_hybrid_kv_cache_manager, vllm_runner
):
"""
The test does a bunch of assignments "x1 = 10\nx2 = 33\n..." and then
"""The test does a bunch of assignments "x1 = 10\nx2 = 33\n..." and then
asks for value of one of them (which is outside the sliding window).
If we tell it upfront which we are going to be looking for, then
it answers correctly (mostly).
@@ -112,14 +111,14 @@ def test_hybrid_kv_cache_manager_output_equivalence(model, vllm_runner):
def check_length(prompts: list[str], llm: LLM, sliding_window: int):
"""
Check if the prompt length is valid, i.e., longer than the sliding window
"""Check if the prompt length is valid, i.e., longer than the sliding window
size and shorter than the model's max length.
Args:
prompts: list of prompts
llm: LLM object
sliding_window: Sliding window size
"""
tokenizer = llm.get_tokenizer()
max_model_len = llm.llm_engine.model_config.max_model_len
@@ -17,9 +17,7 @@ SEED = 42
@pytest.fixture
def test_prompts():
"""
Adapted from tests/v1/e2e/spec_decode/utils.py
"""
"""Adapted from tests/v1/e2e/spec_decode/utils.py."""
prompt_types = ["repeat", "sentence"]
# Setting higher num prompts increases the chance of numerics mismatch
# due to matrix multiplication numerics depending on batch dimension
+14 -19
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Comprehensive end-to-end tests for `min_tokens` in the V1 engine.
"""Comprehensive end-to-end tests for `min_tokens` in the V1 engine.
Addresses #21950: verify and add CI coverage.
@@ -28,7 +27,7 @@ GREEDY = 0.0 # Deterministic generation for consistent testing
class MinTokensTestCase:
"""Data class for min_tokens test scenarios"""
"""Data class for min_tokens test scenarios."""
def __init__(
self,
@@ -161,7 +160,7 @@ MIN_TOKENS_TEST_CASES = [
@pytest.fixture(scope="module")
def llm_v1():
"""Create V1 LLM instance for testing"""
"""Create V1 LLM instance for testing."""
llm = LLM(
model=TEST_MODEL,
tensor_parallel_size=1,
@@ -173,7 +172,8 @@ def llm_v1():
def test_min_tokens_with_terminal_structured_output(llm_v1: LLM):
"""A grammar whose only legal continuation is EOS must stop under
min_tokens instead of sampling outside the grammar."""
min_tokens instead of sampling outside the grammar.
"""
params = SamplingParams(
temperature=GREEDY,
min_tokens=20,
@@ -193,7 +193,7 @@ def test_min_tokens_with_terminal_structured_output(llm_v1: LLM):
def get_token_count(output: RequestOutput) -> int:
"""Extract token count from LLM output"""
"""Extract token count from LLM output."""
if not output.outputs:
return 0
return len(output.outputs[0].token_ids)
@@ -202,7 +202,7 @@ def get_token_count(output: RequestOutput) -> int:
def assert_min_tokens_satisfied(
output: RequestOutput, test_case: MinTokensTestCase
) -> None:
"""Assert that min_tokens requirement is satisfied"""
"""Assert that min_tokens requirement is satisfied."""
token_count = get_token_count(output)
stop_reason = output.outputs[0].stop_reason if output.outputs else "no output"
@@ -228,8 +228,7 @@ def assert_min_tokens_satisfied(
ids=lambda tc: tc.name,
)
def test_min_tokens_comprehensive(llm_v1: LLM, test_case: MinTokensTestCase):
"""
Comprehensive test for min_tokens functionality in V1 engine.
"""Comprehensive test for min_tokens functionality in V1 engine.
This test covers all critical scenarios for min_tokens:
- Basic functionality (should work)
@@ -240,6 +239,7 @@ def test_min_tokens_comprehensive(llm_v1: LLM, test_case: MinTokensTestCase):
Args:
llm_v1: V1 LLM instance
test_case: Test scenario parameters
"""
# Known failing cases are handled via param-level xfail marks above.
@@ -279,8 +279,7 @@ def test_min_tokens_comprehensive(llm_v1: LLM, test_case: MinTokensTestCase):
def test_min_tokens_basic_functionality(llm_v1: LLM):
"""
Test basic min_tokens functionality without stop conditions.
"""Test basic min_tokens functionality without stop conditions.
This is a baseline test that should always pass and validates
that min_tokens works correctly in the simple case.
@@ -302,8 +301,7 @@ def test_min_tokens_basic_functionality(llm_v1: LLM):
strict=False,
)
def test_min_tokens_stop_strings_bug(llm_v1: LLM):
"""
Test the specific bug where stop strings bypass min_tokens.
"""Test the specific bug where stop strings bypass min_tokens.
This test specifically reproduces the bug Calvin is fixing in PR #22014.
It should fail until that fix is merged.
@@ -353,8 +351,7 @@ def test_min_tokens_stop_strings_bug(llm_v1: LLM):
strict=False,
)
def test_min_tokens_stop_strings_guaranteed_early_trigger(llm_v1: LLM):
"""
Guaranteed test for stop strings bypassing min_tokens bug.
"""Guaranteed test for stop strings bypassing min_tokens bug.
Strategy: Use very low temperature and multiple common stop strings
to virtually guarantee early detection, combined with long min_tokens
@@ -408,8 +405,7 @@ def test_min_tokens_stop_strings_guaranteed_early_trigger(llm_v1: LLM):
strict=False,
)
def test_min_tokens_eos_behavior(llm_v1: LLM):
"""
Verify EOS handling with and without min_tokens.
"""Verify EOS handling with and without min_tokens.
- Without min_tokens: expect early EOS -> finish_reason == "stop",
stop_reason is None, and generated tokens < max_tokens (25).
@@ -492,8 +488,7 @@ def test_min_tokens_eos_behavior(llm_v1: LLM):
def test_min_tokens_validation():
"""
Test that SamplingParams correctly validates min_tokens parameters.
"""Test that SamplingParams correctly validates min_tokens parameters.
This tests the parameter validation logic in SamplingParams.
"""
@@ -71,7 +71,6 @@ def retrieve_chunks(self):
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA not available")
def test_pooling_chunked_prefill(vllm_runner, monkeypatch):
"""Test chunked prefill for pooling models with LastPool."""
with monkeypatch.context() as m:
m.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
model_id = "Qwen/Qwen3-Embedding-0.6B"
@@ -124,7 +123,6 @@ def test_pooling_chunked_prefill(vllm_runner, monkeypatch):
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA not available")
def test_pooling_prefix_cache(vllm_runner, monkeypatch):
"""Test chunked prefill for pooling models with LastPool."""
verses = prompt.split("\n\n")
with monkeypatch.context() as m:
@@ -7,7 +7,8 @@ slightly different kernel/collective states (measured: same-state boots are
bitwise identical across the two modes; cross-state boots differ by up to
~0.25 logprob and can flip a near-tie token). A real sharding bug — rows
routed to the wrong request, misaligned gathers — produces many-nats
divergence and scrambled top-k sets, far outside these bounds."""
divergence and scrambled top-k sets, far outside these bounds.
"""
import pytest
@@ -106,7 +107,8 @@ def _assert_logprob_dicts_close(ref_lps, out_lps, what: str) -> None:
@multi_gpu_test(num_gpus=2)
def test_sharded_sampling_outputs_match(monkeypatch: pytest.MonkeyPatch):
"""Generation, logprobs, and prompt logprobs match between batch-sharded
sampling and the replicated fallback, up to measured boot-state noise."""
sampling and the replicated fallback, up to measured boot-state noise.
"""
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
# Required for the collective_rpc mode check below.
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
+1 -2
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
End-to-end tests for the streaming input feature in AsyncLLM.
"""End-to-end tests for the streaming input feature in AsyncLLM.
These tests verify that:
1. Streaming inputs work correctly with bunched inputs (queued)
@@ -11,7 +11,8 @@ from ...utils import compute_acceptance_len, get_test_prompts
@single_gpu_only
def test_synthetic_acceptance_rate(vllm_runner):
"""Verify that synthetic rejection sampling produces an acceptance
length close to the requested mean acceptance length."""
length close to the requested mean acceptance length.
"""
num_spec_tokens = 3
expected_acceptance_len = 1.875
tolerance = 0.15
@@ -43,8 +43,7 @@ def run_acceptance_length_eval(
chat_template_kwargs: dict[str, Any],
use_mrv2: bool,
):
"""
E2E acceptance-rate validation for speculative decoding.
"""E2E acceptance-rate validation for speculative decoding.
Drives one or more datasets (keyed in ``expected_acceptance_lengths``)
through the spec decode engine and asserts the mean acceptance length
@@ -167,7 +167,8 @@ def test_draft_model_tensor_parallelism(vllm_runner):
@multi_gpu_only(num_gpus=2)
def test_draft_model_engine_args_tensor_parallelism():
"""Ensure the vllm_config for the draft model is created correctly,
and independently of the target model (quantization, TP, etc.)"""
and independently of the target model (quantization, TP, etc.)
"""
engine_args = EngineArgs(
model="Qwen/Qwen3-1.7B-FP8", # <<< tgt quantized
tensor_parallel_size=2,
@@ -198,7 +199,8 @@ def test_draft_model_engine_args_tensor_parallelism():
def _apply_draft_moe_backend(vllm_config: VllmConfig) -> VllmConfig:
"""Replicate SpecDecodeBaseProposer._create_draft_vllm_config logic
so we can test it without instantiating a full proposer."""
so we can test it without instantiating a full proposer.
"""
spec_cfg = vllm_config.speculative_config
if spec_cfg.moe_backend is not None:
return replace(
@@ -223,7 +225,8 @@ def _platform_moe_backend(monkeypatch: pytest.MonkeyPatch) -> MoEBackend:
def test_draft_model_moe_backend_override(monkeypatch: pytest.MonkeyPatch):
"""When moe_backend is set in speculative_config, the draft VllmConfig
should use it while the target keeps its own setting."""
should use it while the target keeps its own setting.
"""
target_moe_backend = _platform_moe_backend(monkeypatch)
engine_args = EngineArgs(
model="Qwen/Qwen3-1.7B",
@@ -248,7 +251,8 @@ def test_draft_model_moe_backend_override(monkeypatch: pytest.MonkeyPatch):
def test_draft_model_moe_backend_inherits_target(monkeypatch: pytest.MonkeyPatch):
"""When moe_backend is not set in speculative_config, the draft should
inherit the target's moe_backend."""
inherit the target's moe_backend.
"""
target_moe_backend = _platform_moe_backend(monkeypatch)
engine_args = EngineArgs(
model="Qwen/Qwen3-1.7B",
@@ -271,7 +275,8 @@ def test_draft_model_moe_backend_inherits_target(monkeypatch: pytest.MonkeyPatch
def test_draft_model_moe_backend_default_auto():
"""When neither target nor draft set moe_backend explicitly, both should
default to 'auto'."""
default to 'auto'.
"""
engine_args = EngineArgs(
model="Qwen/Qwen3-1.7B",
tensor_parallel_size=1,
@@ -292,8 +297,8 @@ def test_draft_model_moe_backend_default_auto():
def test_draft_model_engine_args_rejects_invalid_tp_argname():
"""The user should pass "draft_tensor_parallel_size" rather than
"tensor_parallel_size". We enforce this with validation."""
"tensor_parallel_size". We enforce this with validation.
"""
engine_args = EngineArgs(
model="Qwen/Qwen3-1.7B",
tensor_parallel_size=1,
@@ -310,7 +315,8 @@ def test_draft_model_engine_args_rejects_invalid_tp_argname():
def assert_draft_model_correctness(args: ArgsTest, vllm_runner):
"""Compare the outputs using and not using speculative decoding.
In the greedy decoding case, the outputs must match EXACTLY."""
In the greedy decoding case, the outputs must match EXACTLY.
"""
test_prompts: list[Messages] = get_messages(
dataset=args.dataset, n=args.num_prompts
)
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
This script contains:
"""This script contains:
1. test lora with speculative decoding for batch inference
"""
@@ -59,8 +58,7 @@ def test_batch_inference_correctness(
model_setup: tuple[str, str, str, str, int],
vllm_runner,
):
"""
Compare the outputs of a LLM with only Lora and a LLM with both SD and Lora.
"""Compare the outputs of a LLM with only Lora and a LLM with both SD and Lora.
Should be the same and no failure when doing batch inference.
model_setup: (method, model_name, spec_model_name, lora_path, tp_size)
"""
+1 -2
View File
@@ -28,8 +28,7 @@ def _run_eagle_correctness(
attn_backend: str,
vllm_runner,
):
"""
Compare the outputs of an original LLM and a speculative LLM
"""Compare the outputs of an original LLM and a speculative LLM
which should be the same when using eagle speculative decoding.
"""
method, model_name, spec_model_name, tp_size = model_setup
+1 -2
View File
@@ -157,8 +157,7 @@ def test_mtp_correctness(
expected_accuracy_threshold: float,
vllm_runner,
):
"""
Compare the outputs of a original LLM and a speculative LLM
"""Compare the outputs of a original LLM and a speculative LLM
which should be the same when using MTP speculative decoding. Due to some variance
in the engine, it is possible for some outputs to differ, so we expect that at least
6/10 output tokens match exactly, and that the GSM8k accuracy is above a precomputed
@@ -64,8 +64,7 @@ def test_ngram_gpu_default_with_async_scheduling(
async_scheduling: bool,
vllm_runner,
):
"""
Test ngram_gpu speculative decoding (k=3) correctness with and without
"""Test ngram_gpu speculative decoding (k=3) correctness with and without
async scheduling, validated via GSM8K accuracy.
Uses Qwen/Qwen3-8B (ref GSM8K accuracy: 87%-92%).
"""
@@ -99,8 +98,7 @@ def test_suffix_decoding_acceptance(
model_name: str,
vllm_runner,
):
"""
Check that suffix decoding caching takes effect and improves acceptance
"""Check that suffix decoding caching takes effect and improves acceptance
lengths and acceptance rates over multiple runs of the same prompts.
"""
test_prompts = get_test_prompts(mm_enabled=False)
@@ -32,8 +32,7 @@ def test_speculators_model_integration(
expected_accuracy_threshold: float,
vllm_runner,
):
"""
Test that speculators models work with the simplified integration.
"""Test that speculators models work with the simplified integration.
This verifies the `vllm serve <speculator-model>` use case where
speculative config is automatically detected from the model config
@@ -28,7 +28,8 @@ def test_mtp_sharded_sampling_equivalence(monkeypatch: pytest.MonkeyPatch):
sampling under MTP spec decoding: the collectives move the same logits
bytes, Gumbel keys derive from (request slot, position, seed), and slot
assignment is rank-deterministic. Both runs here are spec decode with
identical math, so outputs must match exactly."""
identical math, so outputs must match exactly.
"""
tp_size = 2
_skip_if_insufficient_gpus_for_tp(tp_size)
model_name = "Qwen/Qwen3.5-0.8B-Base"
+2 -2
View File
@@ -67,8 +67,8 @@ def test_mtp_speculative_mixed_batch_short_prefill(
vllm_runner, model_name, enable_prefix_caching
):
"""Test to ensure MTP speculative decoding correctly handles
short prefill chunks that fall below the reorder_batch_threshold."""
short prefill chunks that fall below the reorder_batch_threshold.
"""
# Set so large that both prefills will be classified as decodes in a mixed batch
# note, with prefix caching we require chunk_size >= mamba_block_size
chunk_size = 256 if not enable_prefix_caching else 2048
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
EPD Correctness Test
"""EPD Correctness Test.
Tests that EPD (Encoder-Prefill-Decode) disaggregation produces the same
outputs as a baseline single instance.
@@ -104,6 +103,7 @@ def check_vllm_server(url: str, timeout=5, retries=10) -> bool:
Returns:
True if the server is ready, False otherwise
"""
for attempt in range(retries):
try:
@@ -138,6 +138,7 @@ def run_chat_completion(
Returns:
Generated text content
"""
with openai.OpenAI(
api_key="EMPTY", base_url=base_url, timeout=120, max_retries=0
@@ -381,7 +381,8 @@ def test_pin_if_ready_none_when_absent():
def test_pin_if_ready_returns_unpinned_entry_when_not_ready():
"""The producer needs "present but not ready" told apart from "absent":
the first is NACKed as retryable, the second as a miss."""
the first is NACKed as retryable, the second as a miss.
"""
cache = EmbeddingCache(num_blocks=4)
cache.alloc("h", 1) # not-ready
entry = cache.pin_if_ready("h")
@@ -131,7 +131,8 @@ def test_has_cache_item_false_when_not_consumer(monkeypatch):
def test_connector_keys_on_identifier_not_mm_hash(monkeypatch):
"""The connector must key the encoder cache on feature.identifier (what
has_cache_item is called with), NOT feature.mm_hash."""
has_cache_item is called with), NOT feature.mm_hash.
"""
s = _make_scheduler(monkeypatch)
req = _Request([_Feature("PROC_KEY", length=1, identifier="ENC_KEY")])
s.update_state_after_alloc(req, 0)
@@ -201,7 +202,8 @@ def test_loads_only_serves_hashes_touched_this_step(monkeypatch):
def test_repeated_reload_same_step_loads_once(monkeypatch):
"""The same mm_hash requested twice in one step must appear in
meta.loads exactly once."""
meta.loads exactly once.
"""
s = _make_scheduler(monkeypatch)
_seed_cached(s, "a", n_blocks=2)
@@ -224,7 +226,8 @@ def test_load_not_emitted_for_uncached_entry(monkeypatch):
def test_load_pin_protects_blocks_until_completion(monkeypatch):
"""Loaded blocks stay pinned until the worker reports the load memcpy
complete. Observable: a save that needs eviction cannot reclaim a
still-pinned loaded entry, but can once the completion report arrives."""
still-pinned loaded entry, but can once the completion report arrives.
"""
# 2 blocks, both occupied by "a" (ready).
s = _make_scheduler(monkeypatch, num_blocks=2)
_seed_cached(s, "a", n_blocks=2)
@@ -251,7 +254,8 @@ def test_load_pin_protects_blocks_until_completion(monkeypatch):
def _can_evict(s: ECCPUScheduler, mm_hash: str, n_blocks: int) -> bool:
"""Whether a new save of `mm_hash` can claim `n_blocks`, i.e. whether the
blocks currently held by other entries are reclaimable."""
blocks currently held by other entries are reclaimable.
"""
s.update_state_after_alloc(_Request([_Feature(mm_hash, length=n_blocks)]), 0)
meta = s.build_connector_meta(scheduler_output=None)
return mm_hash in meta.saves
@@ -311,7 +315,8 @@ def test_late_report_does_not_release_a_later_load_of_same_hash(monkeypatch):
def test_region_full_skips_save_and_never_blocks(monkeypatch):
"""When the region is fully occupied by pinned entries, new saves are
silently skipped and ensure_cache_available never blocks."""
silently skipped and ensure_cache_available never blocks.
"""
s = _make_scheduler(monkeypatch, num_blocks=1)
_seed_cached(s, "pinned", n_blocks=1)
s._cache.pin("pinned")
@@ -330,7 +335,8 @@ def test_region_full_skips_save_and_never_blocks(monkeypatch):
def test_producer_only_never_emits_loads(monkeypatch):
"""A producer-only scheduler must never populate meta.loads, even when
entries are ready."""
entries are ready.
"""
s = _make_scheduler(monkeypatch, ec_role="ec_producer")
req = _Request([_Feature("h1", length=1)])
@@ -372,7 +378,8 @@ def test_consumer_only_has_cache_item(monkeypatch):
def test_save_not_emitted_for_already_cached_entry(monkeypatch):
"""An entry already in the cache (from a prior save) must not trigger
a second allocation or appear in meta.saves again."""
a second allocation or appear in meta.saves again.
"""
s = _make_scheduler(monkeypatch)
req = _Request([_Feature("h1", length=1)])
@@ -393,7 +400,8 @@ def test_save_not_emitted_for_already_cached_entry(monkeypatch):
def test_update_connector_output_ignores_foreign_meta(monkeypatch):
"""A worker output whose payload is not an ECCPUWorkerMetadata is a
no-op — no cache mutation, no crash."""
no-op — no cache mutation, no crash.
"""
s = _make_scheduler(monkeypatch)
_seed_cached(s, "a", n_blocks=1)
@@ -407,7 +415,8 @@ def test_update_connector_output_ignores_foreign_meta(monkeypatch):
def test_update_connector_output_ignores_unknown_report(monkeypatch):
"""Reports for a never-seen save hash or a transfer id this scheduler never
dispatched are dropped rather than mutating unrelated state."""
dispatched are dropped rather than mutating unrelated state.
"""
s = _make_scheduler(monkeypatch)
s.update_connector_output(_WorkerOutput(saves=["gone"], loads=[4242]))
assert s.has_cache_item("gone") is False
@@ -419,7 +428,8 @@ def test_update_connector_output_ignores_unknown_report(monkeypatch):
def test_has_pending_push_work_tracks_inflight_save(monkeypatch):
"""A dispatched-but-unconfirmed save keeps push work pending until the
completion report marks it ready."""
completion report marks it ready.
"""
s = _make_scheduler(monkeypatch)
assert s.has_pending_push_work() is False
@@ -434,7 +444,8 @@ def test_has_pending_push_work_tracks_inflight_save(monkeypatch):
def test_has_pending_push_work_tracks_inflight_load(monkeypatch):
"""A dispatched-but-unconfirmed load keeps push work pending until the
unpin completion report arrives."""
unpin completion report arrives.
"""
s = _make_scheduler(monkeypatch)
_seed_cached(s, "a", n_blocks=1)
assert s.has_pending_push_work() is False # ready + unpinned
@@ -453,7 +464,8 @@ def test_has_pending_push_work_tracks_inflight_load(monkeypatch):
def test_shutdown_disables_roles_and_cleans_region(monkeypatch):
"""After shutdown the scheduler serves no items and the region is
cleaned up."""
cleaned up.
"""
s = _make_scheduler(monkeypatch, num_blocks=2)
_seed_cached(s, "a", n_blocks=2)
assert s.has_cache_item("a") is True
@@ -98,7 +98,8 @@ def test_second_instance_opens_existing_file_and_shares_memory():
def test_only_creator_unlinks_file_on_cleanup():
"""Critical contract: if the non-creator unlinks, the creator's mmap path
becomes a dangling backing file and a third opener would create a new one
out from under the creator."""
out from under the creator.
"""
instance_id = str(uuid.uuid4())
r1 = ECSharedRegion(
engine_id=instance_id,
@@ -215,7 +216,8 @@ def test_madvise_einval_selects_fallback_for_whole_region(monkeypatch):
def test_non_creator_does_not_probe_or_populate(monkeypatch):
"""Only the creator pre-faults. A second opener must not probe the advice
nor re-touch pages the creator already populated."""
nor re-touch pages the creator already populated.
"""
from vllm.distributed.ec_transfer.ec_connector.cpu import ec_shared_region as esr
instance_id = str(uuid.uuid4())
@@ -312,7 +314,8 @@ def test_madvise_unexpected_oserror_propagates(monkeypatch):
def test_pin_memory_success_sets_flag(region):
"""When cudaHostRegister returns 0, _is_pinned flips to True
and cleanup will correspondingly call cudaHostUnregister."""
and cleanup will correspondingly call cudaHostUnregister.
"""
fake_cudart = MagicMock()
success = MagicMock()
success.value = 0
@@ -332,7 +335,8 @@ def test_pin_memory_success_sets_flag(region):
def test_pin_memory_failure_leaves_flag_false():
"""If cudaHostRegister fails (non-zero), don't pretend it succeeded —
cleanup must NOT call cudaHostUnregister on memory we never registered."""
cleanup must NOT call cudaHostUnregister on memory we never registered.
"""
r = _make_region()
try:
fake_cudart = MagicMock()
@@ -363,7 +363,8 @@ def test_save_caches_coalesces_only_adjacent_blocks(
def test_save_caches_noop_when_mm_hash_not_in_saves(make_worker):
"""When the scheduler hasn't pre-allocated blocks for ``mm_hash``,
``save_caches`` + ``flush_saves`` is a pure no-op."""
``save_caches`` + ``flush_saves`` is a pure no-op.
"""
worker = make_worker()
sentinel = 0x42
worker._region.blocks.fill_(sentinel)
@@ -381,7 +382,8 @@ def test_save_caches_noop_when_mm_hash_not_in_saves(make_worker):
)
def test_save_caches_noop_for_non_save_rank(make_worker, tp_rank, pcp_rank):
"""Only TP rank 0 + PCP rank 0 writes to mmap. All other combinations
must skip the save entirely."""
must skip the save entirely.
"""
worker = make_worker(tp_rank=tp_rank, pcp_rank=pcp_rank)
sentinel = 0x42
worker._region.blocks.fill_(sentinel)
@@ -396,7 +398,8 @@ def test_save_caches_noop_for_non_save_rank(make_worker, tp_rank, pcp_rank):
def test_save_caches_raises_when_allocated_blocks_too_small(make_worker):
"""``save_caches`` must raise ``AssertionError`` when the encoder output
is larger than the allocated block space."""
is larger than the allocated block space.
"""
worker = make_worker()
src = torch.zeros(3 * _HIDDEN_DIM, dtype=_DTYPE)
with pytest.raises(AssertionError, match="exceeds allocated blocks"):
@@ -549,7 +552,8 @@ def test_start_load_caches_noop_when_loads_is_empty(make_worker):
def test_start_load_caches_skips_cached_and_loads_new_in_same_step(make_worker):
"""Every hash in ``meta.loads`` is copied from mmap, including one whose
key is already resident in ``encoder_cache`` — there is no worker-side
skip."""
skip.
"""
worker = make_worker()
n_blocks = 3
src_orig = torch.arange(n_blocks * _HIDDEN_DIM, dtype=_DTYPE).reshape(
@@ -778,7 +782,8 @@ def test_buffer_pool_is_reused_across_save_steps(make_worker):
@_requires_accelerator
def test_buffer_pool_is_reused_across_load_steps(make_worker):
"""Once a load copy completes its descriptor buffers return to the pool and
are reused by the next call — no reallocation."""
are reused by the next call — no reallocation.
"""
worker = make_worker()
worker._region.blocks[0].fill_(0x01)
worker._region.blocks[1].fill_(0x02)
@@ -849,7 +854,8 @@ def test_init_pins_memory_when_available(pin_available, expected_pinned):
def test_shutdown_calls_region_cleanup_and_swallows_errors(caplog_vllm):
"""``shutdown`` must always call ``region.cleanup`` — and must never
raise."""
raise.
"""
worker = object.__new__(ECCPUWorker)
mock_region = Mock(spec=ECSharedRegion)
worker._region = mock_region
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
E2E test for the ECCPUConnector.
"""E2E test for the ECCPUConnector.
Verifies:
- Accuracy: outputs from EC CPU cache match fresh encoder computation.
@@ -1,8 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Unit tests for ECExampleConnector.
"""
"""Unit tests for ECExampleConnector."""
import os
from unittest.mock import Mock, patch
@@ -43,7 +43,8 @@ def assign(proxy):
def _drive(assign, e_urls, counts):
"""Feed a sequence of request item-counts through the cursor, as
fanout_encoder_primer does one request at a time."""
fanout_encoder_primer does one request at a time.
"""
cursor = 0
all_urls = []
for count in counts:
+3 -2
View File
@@ -40,7 +40,8 @@ def test_metadata_saves_dict_operations():
def test_metadata_loads_carry_a_transfer_id_and_blocks():
"""Each load entry is a (transfer_id, block_ids) pair, so the worker can
report the transfer rather than the mm_hash."""
report the transfer rather than the mm_hash.
"""
meta = ECCPUConnectorMetadata()
meta.loads["mm_hash_1"] = (0, [5, 6, 7])
@@ -54,7 +55,7 @@ def test_metadata_loads_carry_a_transfer_id_and_blocks():
def test_metadata_saves_and_loads_are_independent():
"""saves and loads dicts are independent."""
"""Saves and loads dicts are independent."""
meta = ECCPUConnectorMetadata()
meta.saves["key"] = [1, 2]
@@ -79,7 +79,8 @@ def test_request_finished_announces_not_ready_entry(monkeypatch):
"""A save whose GPU->mmap copy hasn't been confirmed complete yet is
still announced: the entry can't be evicted before it's ready, and a read
arriving too early is NACKed NACK_NOT_READY for the consumer to retry —
protecting it is not this method's job."""
protecting it is not this method's job.
"""
s = _sched_gate_off(monkeypatch)
s._nixl_enabled = True
s._peer_host, s._peer_port = "1.2.3.4", 5601
+4 -3
View File
@@ -129,7 +129,8 @@ def test_producer_session_grant_nacks_missing_entry():
def test_producer_session_grant_nacks_not_ready_entry_as_retryable():
"""An announced entry whose save is still in flight is distinguished from a
miss, so the consumer retries instead of falling back to local compute."""
miss, so the consumer retries instead of falling back to local compute.
"""
cache = EmbeddingCache(num_blocks=8)
cache.alloc("h1", 2) # not ready
s = _make_producer_session(cache)
@@ -329,7 +330,6 @@ def test_consumer_xfer_quarantined_exception_settles():
def test_consumer_xfer_quarantined_timeout_evicts_blocks():
"""Quarantine timeout should release handle and return READ_FAILED."""
data = _make_data(xfer_state="PROC")
x = _started(data)
# First poll transitions to QUARANTINED (deadline extended by 60s)
@@ -438,7 +438,8 @@ def test_consumer_session_poll_nack_goes_to_tombstoned():
def test_consumer_session_poll_not_ready_nack_goes_to_retryable():
"""A not-ready NACK must not tombstone: tombstoning admits the request with
no embedding, which is fatal once the media has been rewritten away."""
no embedding, which is fatal once the media has been rewritten away.
"""
import msgspec
s = _make_consumer_session()
+1
View File
@@ -45,6 +45,7 @@ def create_ec_vllm_config(
Returns:
A `VllmConfig`-specced mock carrying the configs above.
"""
parallel_config = ParallelConfig(
tensor_parallel_size=tensor_parallel_size,
+4 -3
View File
@@ -24,12 +24,12 @@ EngineCorePromptLogprobsType = tuple[torch.Tensor, torch.Tensor]
def _build_test_vectors_no_logprobs() -> DummyOutputProcessorTestVectors:
"""Generate output processor dummy test vectors, without logprobs
"""Generate output processor dummy test vectors, without logprobs.
Returns:
DummyOutputProcessorTestVectors instance with no logprobs
"""
"""
tokenizer = AutoTokenizer.from_pretrained(TOKENIZER_NAME)
vllm_config = EngineArgs(model=TOKENIZER_NAME).create_engine_config()
# Tokenize prompts under test & create dummy generated tokens
@@ -62,10 +62,11 @@ def _build_test_vectors_no_logprobs() -> DummyOutputProcessorTestVectors:
@pytest.fixture
def dummy_test_vectors() -> DummyOutputProcessorTestVectors:
"""Generate output processor dummy test vectors, with logprobs
"""Generate output processor dummy test vectors, with logprobs.
Returns:
DummyOutputProcessorTestVectors instance with logprobs
"""
# Build dummy test vectors without logprobs
dtv = _build_test_vectors_no_logprobs()
+3 -7
View File
@@ -1,8 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Test for the fix in PR #29987: Eagerly abort cancelled final-step requests.
"""Test for the fix in PR #29987: Eagerly abort cancelled final-step requests.
This test verifies that when a request is aborted during its final execution
step (when it would naturally complete), it is properly marked as aborted
@@ -52,8 +51,7 @@ class DummyKVConnectorMetadata(KVConnectorMetadata):
class DummyKVConnector(KVConnectorBase_V1):
"""
Dummy KV connector that captures request finish statuses to a file.
"""Dummy KV connector that captures request finish statuses to a file.
This is used to verify the fix - without the fix, a request aborted
during its final step would be captured as FINISHED_LENGTH_CAPPED
instead of FINISHED_ABORTED.
@@ -144,8 +142,7 @@ KVConnectorFactory.register_connector(
@pytest.mark.parametrize("async_scheduling", [False, True])
@pytest.mark.asyncio
async def test_abort_during_final_step(async_scheduling: bool):
"""
Test that a request aborted during its final execution step is treated as
"""Test that a request aborted during its final execution step is treated as
aborted rather than completed.
This test:
@@ -165,7 +162,6 @@ async def test_abort_during_final_step(async_scheduling: bool):
With the fix, _process_aborts_queue() runs before update_from_output(), so the
abort takes precedence and the KV connector sees FINISHED_ABORTED.
"""
# Create three temporary files:
# 1. ready_file: deleted by execute_model to signal it has started
# 2. block_file: execute_model waits for this to be deleted
-2
View File
@@ -457,7 +457,6 @@ async def test_customize_loggers(monkeypatch):
If a customized logger is provided at the init, it should
be added to the default loggers.
"""
with ExitStack() as after:
with set_default_torch_num_threads(1):
engine = AsyncLLM.from_engine_args(
@@ -642,7 +641,6 @@ async def test_check_health():
@pytest.mark.asyncio
async def test_abort_final_output(output_kind: RequestOutputKind):
"""Test that abort() returns a final output with correct information."""
with ExitStack() as after:
with set_default_torch_num_threads(1):
engine = AsyncLLM.from_engine_args(TEXT_ENGINE_ARGS)
@@ -68,7 +68,8 @@ class _StubEngineCoreActor(EngineCoreActorMixin):
# in ``tests/entrypoints/test_api_server_process_manager.py``.
def _bind_and_report_worker(listen_address, sock, args, client_config):
"""Bind ROUTER/PULL with a kernel-assigned port, report the actual
endpoints back via ``actual_address_pipe``, then exit."""
endpoints back via ``actual_address_pipe``, then exit.
"""
ctx = zmq.Context()
try:
in_sock = make_zmq_socket(
+12 -16
View File
@@ -193,8 +193,7 @@ def test_engine_core():
@create_new_process_for_each_test()
def test_engine_core_advanced_sampling():
"""
A basic end-to-end test to verify that the engine functions correctly
"""A basic end-to-end test to verify that the engine functions correctly
when additional sampling parameters, such as top_p, min_tokens, and
presence_penalty, are set.
"""
@@ -242,9 +241,7 @@ def test_engine_core_advanced_sampling():
@create_new_process_for_each_test()
def test_engine_core_concurrent_batches():
"""
Test that the engine can handle multiple concurrent batches.
"""
"""Test that the engine can handle multiple concurrent batches."""
def make_request_with_max_tokens(req_id: str, max_tokens: int) -> EngineCoreRequest:
request = make_request()
@@ -265,7 +262,6 @@ def test_engine_core_concurrent_batches():
non_block=False,
) -> Future[ModelRunnerOutput | None]:
"""Make execute_model non-blocking."""
# DummyExecutor used only for testing async case.
assert non_block
@@ -282,7 +278,6 @@ def test_engine_core_concurrent_batches():
self, grammar_output, non_block=False
) -> Future[ModelRunnerOutput]:
"""Make sample_tokens non-blocking."""
# DummyExecutor used only for testing async case.
assert non_block
@@ -401,10 +396,7 @@ def test_engine_core_concurrent_batches():
@multi_gpu_test(num_gpus=2)
def test_engine_core_tp():
"""
Test engine can initialize worker in tp properly
"""
"""Test engine can initialize worker in tp properly."""
"""Setup the EngineCore."""
engine_args = EngineArgs(
model=MODEL_NAME,
@@ -490,7 +482,7 @@ def test_encoder_instance_zero_kv_cache(
enable_prefix_caching: bool,
use_kv_connector: bool,
):
"""EPD (Encoder-Prefill-Decode) Encoder-cache-specific tests
"""EPD (Encoder-Prefill-Decode) Encoder-cache-specific tests.
This test verifies encoder-only instance initializes with 0 KV cache blocks.
Under EPD disagg mode, Encoder instances (EC producer role) only execute
@@ -615,7 +607,8 @@ def _cadenced_dp_engine_core(
monkeypatch, results: list[tuple[bool, bool]], dp_sync_interval: int = 32
):
"""A bare DPEngineCoreProc whose all-reduce is scripted by `results`;
returns the core and the step numbers at which the all-reduce ran."""
returns the core and the step numbers at which the all-reduce ran.
"""
core = object.__new__(DPEngineCoreProc)
core.dp_group = object()
core.dp_sync_interval = dp_sync_interval
@@ -634,7 +627,8 @@ def _cadenced_dp_engine_core(
def test_dp_sync_interval_default_is_16():
"""Regression pin: lowering the default narrows the mid-wave pause tail
for async RL, where the engine is rarely idle when pause lands."""
for async RL, where the engine is rarely idle when pause lands.
"""
assert ParallelConfig.dp_sync_interval == 16
@@ -652,7 +646,8 @@ def test_dp_sync_interval_normal_wave(monkeypatch, dp_sync_interval: int):
def test_dp_sync_interval_idle_pause_consensus_on_first_step(monkeypatch):
"""A pause of an idle engine arms every rank before its kick-started
first step, so the step-1 sync reaches consensus after one dummy batch
regardless of the configured cadence."""
regardless of the configured cadence.
"""
core, synced = _cadenced_dp_engine_core(
monkeypatch, [(False, True)], dp_sync_interval=32
)
@@ -678,7 +673,8 @@ def _pausable_engine_core_proc() -> EngineCoreProc:
@pytest.mark.parametrize("deferred", [False, True])
def test_pause_synchronizes_device_before_cache_reset(deferred: bool):
"""A resolved pause promises an idle device: the barrier must run before
caches are cleared and before the caller is unblocked."""
caches are cleared and before the caller is unblocked.
"""
core = _pausable_engine_core_proc()
core.engines_running = deferred
order: list[str] = []
+7 -6
View File
@@ -249,7 +249,8 @@ def test_dplb_non_late_interaction_still_uses_lb():
def test_dplb_burst_round_robins_despite_snapshot_rebinds():
"""A stats snapshot rebind wipes the optimistic lb_engines increments;
the exact in-flight floor must keep a burst spreading round-robin."""
the exact in-flight floor must keep a burst spreading round-robin.
"""
client = _make_dplb_client(num_engines=4)
for _ in range(4):
@@ -324,7 +325,8 @@ async def test_dplb_scale_down_routes_after_stale_stats_snapshot():
def test_dplb_snapshot_backpressure_overrides_inflight():
"""An engine reported heavily loaded by the coordinator is avoided even
when this client has routed nothing to it."""
when this client has routed nothing to it.
"""
client = _make_dplb_client(num_engines=2)
client.lb_engines = [[5, 10, 0.0], [0, 0, 0.0]]
@@ -337,7 +339,8 @@ def test_dplb_snapshot_backpressure_overrides_inflight():
def test_dplb_kv_pressure_amplifies_waiting_penalty():
"""A waiting queue on a KV-bound engine (slow drain) is penalized, while
the same queue with low KV usage is not (e.g. transient burst)."""
the same queue with low KV usage is not (e.g. transient burst).
"""
client = _make_dplb_client(num_engines=2)
# Engine 0 has a smaller total but is KV-bound with a queue.
client.lb_engines = [[5, 10, 1.0], [0, 20, 0.2]]
@@ -1392,11 +1395,9 @@ def test_startup_failure(monkeypatch: pytest.MonkeyPatch):
@create_new_process_for_each_test()
def test_engine_core_proc_instantiation_cuda_empty(monkeypatch: pytest.MonkeyPatch):
"""
Test that EngineCoreProc can be instantiated when CUDA_VISIBLE_DEVICES
"""Test that EngineCoreProc can be instantiated when CUDA_VISIBLE_DEVICES
is empty. This ensures the engine frontend does not need access to GPUs.
"""
from vllm.v1.engine.core import EngineCoreProc
from vllm.v1.executor.abstract import Executor
@@ -11,8 +11,7 @@ from vllm.v1.engine.detokenizer import IncrementalDetokenizer
def test_fast_inc_detok_invalid_utf8_err_case():
"""
Test edge case where tokenizer can produce non-monotonic,
"""Test edge case where tokenizer can produce non-monotonic,
invalid UTF-8 output, which breaks the internal state of
tokenizers' DecodeStream.
See https://github.com/vllm-project/vllm/issues/17448.
+2 -1
View File
@@ -82,7 +82,7 @@ def _get_test_sampling_params(
rng = random.Random(seed)
def get_mostly_n_gt1() -> int:
r"""Mostly n \in [2,20], ~1/3 n=1"""
r"""Mostly n \in [2,20], ~1/3 n=1."""
x = rng.randint(0, 28)
if x < 10:
return 1
@@ -125,6 +125,7 @@ def test_parallel_sampling(vllm_model, example_prompts) -> None:
Args:
vllm_model: VllmRunner instance under test.
example_prompt: test fixture providing prompts for testing.
"""
sampling_params_list, n_list = _get_test_sampling_params(example_prompts)
llm: LLM = vllm_model.llm
+2 -1
View File
@@ -30,7 +30,8 @@ def _make_processor(num_logprobs: int) -> LogprobsProcessor:
def test_drops_trailing_sentinel_columns():
"""A request that asked for 3 custom token logprobs but ended up in a
batch padded to width 5 must not surface the trailing -inf entries."""
batch padded to width 5 must not surface the trailing -inf entries.
"""
processor = _make_processor(num_logprobs=3)
sampled = 42
+4 -1
View File
@@ -73,6 +73,7 @@ def _ref_convert_id_to_token(
Returns:
String representation of input token id
"""
return tokenizer.decode([token_id]) or ""
@@ -228,7 +229,8 @@ def test_request_stream_interval_raises_but_not_below_engine_default(
):
"""A per-request stream_interval can raise the interval above the engine
default but not below it (values under the default clamp up), without
altering the generated text."""
altering the generated text.
"""
engine_stream_interval = 5
# Request 0 (below the default) clamps up to 5; request 1 raises it to 10.
request_stream_intervals = [1, 10]
@@ -792,6 +794,7 @@ def test_stop_token(
stop_token_type: "eos_token_id" for EOS, "stop_token_ids" for stop token
ignore_eos: if True, EOS stops are disabled
dummy_test_vectors: dummy engine core outputs and other data structures
"""
model_id = dummy_test_vectors.tokenizer.name_or_path
if model_id != "meta-llama/Llama-3.2-1B":
@@ -14,7 +14,6 @@ MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
def test_preprocess_error_handling(monkeypatch: pytest.MonkeyPatch):
"""Test that preprocessing errors are handled gracefully."""
if current_platform.is_rocm() or current_platform.is_xpu():
pytest.skip(
"Skipped on ROCm/XPU: this test only works with 'fork', "
+14 -8
View File
@@ -54,6 +54,7 @@ def _create_random_top_logprob_test_vector(
Returns:
1D length-`num_logprobs` torch Tensor of float logprob values
"""
return torch.rand(num_logprobs) * (upper - lower) + lower
@@ -79,6 +80,7 @@ def _create_random_top_logprob_test_matrix(
Returns:
2D num_tokens x num_logprobs torch Tensor of float logprob values
"""
return torch.rand(*shape) * (upper - lower) + lower
@@ -90,7 +92,7 @@ def _create_random_top_token_test_vector(
sampled_token_id: int,
adjust_num_logprobs: bool = True,
) -> tuple[torch.Tensor, int]:
"""Create a random vector of top logprob token indices
"""Create a random vector of top logprob token indices.
Use to create fake sample logprobs for testing. The sampled token
ID must always be one of the top logprobs, which this dummy test
@@ -117,8 +119,8 @@ def _create_random_top_token_test_vector(
sampled_token_rank: the rank of sampled_token_id in the vocab
vector when sorted in descending order by
logprob
"""
"""
# Calculate the final number of logprobs required
total_logprobs = num_logprobs + 1 if adjust_num_logprobs else num_logprobs
@@ -148,7 +150,7 @@ def _create_random_top_token_test_matrix(
upper: int,
tokens_list: list[int],
) -> tuple[torch.Tensor, torch.Tensor]:
"""Create a random matrix of top logprob token indices
"""Create a random matrix of top logprob token indices.
Use to create fake prompt logprobs for testing.
@@ -166,6 +168,7 @@ def _create_random_top_token_test_matrix(
- 2D num_tokens x num_logprobs+1 torch Tensor of token ids
- 1D tensor of ranks of prompt tokens in their respective
rows, or random values
"""
num_elements = shape[0] * shape[1]
choice_tensor = torch.randperm(upper - lower)[:num_elements] + lower
@@ -205,6 +208,7 @@ def decode_token(
Returns:
string representation of token
"""
return tokenizer.convert_ids_to_tokens(tok_id)
@@ -214,7 +218,7 @@ def generate_dummy_sample_logprobs(
num_logprobs: int,
tokenizer: PythonBackend,
) -> list[tuple[list[int], list[float], int]]:
"""Generate dummy sample logprobs
"""Generate dummy sample logprobs.
Generate a test data structure which imitates the list of sample logprobs
which would be assembled in the engine core during decode phase.
@@ -224,13 +228,14 @@ def generate_dummy_sample_logprobs(
num_logprobs: return `num_logprobs` or `num_logprobs+1` logprobs per token
tokenizer: model tokenizer to use for detokenization
Returns
Returns:
list of (top token ids vector, logprobs vector, sampled token rank)
Python lists tuples; in each tuple the logprobs and top token ids
vectors have the same length which is either `num_logprobs` or
`num_logprobs+1`. Sampled token rank is the rank (index+1) of the
sampled token within the vocab vector when sorted by logprob in
descending order.
"""
res = []
for sampled_token_id in sampled_tokens_list:
@@ -263,7 +268,7 @@ def generate_dummy_prompt_logprobs_tensors(
num_logprobs: int,
tokenizer: PythonBackend,
) -> LogprobsTensors:
"""Generate dummy prompt logprobs tensors
"""Generate dummy prompt logprobs tensors.
Generate a test data structure which imitates the torch Tensors of prompt
logprobs which would be assembled in the engine core during chunked
@@ -274,10 +279,11 @@ def generate_dummy_prompt_logprobs_tensors(
num_logprobs: return `num_logprobs` logprobs per token
tokenizer: model tokenizer to use for detokenization
Returns
Returns:
Single tuple of (logprobs matrix, top token ids matrix) torch Tensor,
where both matrices have dimensions
num_prompt_tokens x num_logprobs
"""
# For now, assume the whole prompt is processed in one chunk; thus,
# the number of non-`None` prompt logprobs is `len(prompt_tokens_list)-1`.
@@ -308,7 +314,7 @@ def generate_dummy_prompt_logprobs_tensors(
@dataclass
class DummyOutputProcessorTestVectors:
"""Dummy test vectors for output processor tests"""
"""Dummy test vectors for output processor tests."""
tokenizer: GeneralTokenizerType
vllm_config: EngineArgs
@@ -59,6 +59,7 @@ class PredictableLlamaModel(nn.Module, EagleModelMixin):
Returns:
If aux_hidden_state_layers is set: (hidden_states, aux_hidden_states)
Otherwise: hidden_states
"""
# Determine sequence length
if inputs_embeds is not None:
@@ -25,8 +25,7 @@ SAMPLE_PROMPTS = (
def check_vllm_server(url: str, timeout=5, retries=3) -> bool:
"""
Checks if the vLLM server is ready by sending a GET request to the
"""Checks if the vLLM server is ready by sending a GET request to the
/health endpoint.
Args:
@@ -36,6 +35,7 @@ def check_vllm_server(url: str, timeout=5, retries=3) -> bool:
Returns:
bool: True if the server is ready, False otherwise.
"""
for attempt in range(retries):
try:
@@ -81,8 +81,7 @@ def run_simple_prompt(
def main():
"""
This script demonstrates how to accept two optional string arguments
"""This script demonstrates how to accept two optional string arguments
("service_url" and "file_name") from the command line, each with a
default value of an empty string, using the argparse module.
"""
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Integration edge-case tests for MultiConnector (NixlConnector + OffloadingConnector).
"""Integration edge-case tests for MultiConnector (NixlConnector + OffloadingConnector).
Tests cover:
- Output correctness across block-size boundaries (proxy vs direct prefill).
@@ -18,9 +18,7 @@ logger.setLevel(logging.DEBUG)
@asynccontextmanager
async def lifespan(app: FastAPI):
"""
Lifespan context manager to handle startup and shutdown events.
"""
"""Lifespan context manager to handle startup and shutdown events."""
# Startup: Initialize client pools for prefiller and decoder services
app.state.prefill_clients = []
app.state.decode_clients = []
@@ -132,8 +130,7 @@ def parse_args():
def get_next_client(app, service_type: str):
"""
Get the next client in round-robin fashion.
"""Get the next client in round-robin fashion.
Args:
app: The FastAPI app instance
@@ -141,6 +138,7 @@ def get_next_client(app, service_type: str):
Returns:
The next client to use
"""
if service_type == "prefill":
client_idx = next(app.state.prefill_iterator)
@@ -155,9 +153,7 @@ def get_next_client(app, service_type: str):
async def send_request_to_service(
client_info: dict, endpoint: str, req_data: dict, request_id: str
):
"""
Send a request to a service using a client from the pool.
"""
"""Send a request to a service using a client from the pool."""
req_data = req_data.copy()
req_data["kv_transfer_params"] = {
"do_remote_decode": True,
@@ -200,9 +196,7 @@ async def send_request_to_service(
async def stream_service_response(
client_info: dict, endpoint: str, req_data: dict, request_id: str
):
"""
Asynchronously stream response from a service using a client from the pool.
"""
"""Asynchronously stream response from a service using a client from the pool."""
headers = {
"Authorization": f"Bearer {os.environ.get('OPENAI_API_KEY')}",
"X-Request-Id": request_id,
@@ -80,7 +80,8 @@ def _split_hnd_cache(spec) -> torch.Tensor:
def _packed_nhd_cache(spec) -> torch.Tensor:
"""Logical (num_blocks, heads, block_size, 2 * head_size) over an NHD
physical layout — the FlashAttention/FlashInfer/Triton/Flex form."""
physical layout — the FlashAttention/FlashInfer/Triton/Flex form.
"""
return torch.zeros(
NUM_BLOCKS,
spec.block_size,
@@ -863,7 +863,8 @@ def test_parallelism_agnostic_excluded(kv_cache_groups: list[KVCacheGroupSpec]):
)
def test_canonical_layout_gate(kv_cache_groups, certified):
"""The canonical layout certifies portability group by group; none of
these shapes are portable in the direct layout."""
these shapes are portable in the direct layout.
"""
assert not _parallelism_agnostic(kv_cache_groups)
assert _parallelism_agnostic(kv_cache_groups, canonical=True) is certified
@@ -871,7 +872,8 @@ def test_canonical_layout_gate(kv_cache_groups, certified):
def test_canonical_layout_certifies_v2_model_runner():
"""Canonical bytes are certified per layer against live tensor strides at
registration, so the static gate must not depend on the model-runner
version — the v2 runner is the case the canonical layout exists for."""
version — the v2 runner is the case the canonical layout exists for.
"""
groups = _groups(_full_attention_spec())
assert _parallelism_agnostic(groups, canonical=True, v2=True)
@@ -117,7 +117,8 @@ def _spec_cls_with_metric_definitions(
metric_definitions: dict[str, Any],
) -> type:
"""Build a fake offloading spec class reporting the given metric
definitions, so tests don't need to patch the real CPU spec."""
definitions, so tests don't need to patch the real CPU spec.
"""
class _FakeOffloadingSpec:
@staticmethod
@@ -187,7 +188,8 @@ def test_build_kv_connector_stats_with_empty_dict():
def test_build_kv_connector_stats_reconstructs_offload_stats():
"""Test that OffloadingConnector stats are properly reconstructed with
correct data."""
correct data.
"""
serialized_data = {
_StatsKey.TYPES: {
LOAD_BYTES: _MetricType.COUNTER,
@@ -1583,7 +1583,8 @@ _SCAN_BEHAVIOR = {
@pytest.mark.parametrize("result", list(LookupResult))
def test_scan_behavior_declared_for_every_lookup_result(result: LookupResult):
"""Whether a result keeps a sliding-window streak alive decides how wide
the demanded chunk set gets, so every member needs deliberate behavior."""
the demanded chunk set gets, so every member needs deliberate behavior.
"""
assert result in _SCAN_BEHAVIOR, f"{result} has no declared scan behavior"
expected_lookups, expected_end = _SCAN_BEHAVIOR[result]
@@ -2034,7 +2035,8 @@ def test_request_level_policy_stores_all_blocks(request_runner, async_scheduling
def test_loads_do_not_populate_fence_index(request_runner):
"""Loads don't populate _block_id_to_pending_jobs (protected by
delay_free_blocks while in flight)."""
delay_free_blocks while in flight).
"""
runner = request_runner(
blocks_per_chunk=3,
block_size=4,
@@ -2104,7 +2106,8 @@ def test_fence_at_update_state_after_alloc(request_runner):
def test_fence_at_build_store_jobs(request_runner):
"""A new prefill (no load -> update_state_after_alloc returns early)
reusing a finished request's pending-store block is flushed by
_build_store_jobs's fence."""
_build_store_jobs's fence.
"""
runner = request_runner(
blocks_per_chunk=1,
block_size=4,
@@ -2154,7 +2157,8 @@ def test_fence_at_build_store_jobs(request_runner):
@pytest.mark.parametrize("async_scheduling", [True, False])
def test_complete_store_called_per_job(request_runner, async_scheduling: bool):
"""complete_store fires per-job, not deferred to request finish.
Each call carries only that store's keys."""
Each call carries only that store's keys.
"""
tokens_per_block = 4
blocks_per_chunk = 3
tokens_per_chunk = tokens_per_block * blocks_per_chunk
@@ -2383,7 +2387,8 @@ def test_offload_prompt_only(request_runner, async_scheduling: bool):
@pytest.mark.parametrize("async_scheduling", [True, False])
def test_reset_cache(request_runner, async_scheduling: bool):
"""reset_cache flushes in-flight loads, calls manager.reset_cache(), resets
next_stored_chunk_idx for active requests and clears job tracking."""
next_stored_chunk_idx for active requests and clears job tracking.
"""
block_size = 4
blocks_per_chunk = 3
tokens_per_chunk = block_size * blocks_per_chunk
@@ -2929,7 +2934,8 @@ def test_stale_sliding_window_block_after_prepare_store_failure(
@pytest.mark.parametrize("async_scheduling", [True, False])
def test_skip_reading_prefix_cache(request_runner, async_scheduling: bool):
"""When skip_reading_prefix_cache=True, the offloading connector must not
load any blocks from CPU even if a matching prefix is cached there."""
load any blocks from CPU even if a matching prefix is cached there.
"""
block_size = 4
blocks_per_chunk = 3
tokens_per_chunk = block_size * blocks_per_chunk
@@ -3023,7 +3029,8 @@ def test_max_load_tokens_limits_external_load(request_runner, async_scheduling:
class TestEagle:
"""Tests for Eagle/MTP speculative decoding support in the offloading
connector scheduler — both _lookup() unit tests and integration tests."""
connector scheduler — both _lookup() unit tests and integration tests.
"""
# -------------------------------------------------------------------
# Helpers
@@ -3565,7 +3572,8 @@ class TestEagle:
self, request_runner, async_scheduling: bool
):
"""Eagle sliding-window group stores all prompt blocks but excludes
the trailing chunk while decoding."""
the trailing chunk while decoding.
"""
block_size = 4
sliding_window = 8
num_gpu_blocks = 100
@@ -3617,7 +3625,8 @@ class TestEagle:
self, request_runner, async_scheduling: bool
):
"""An eagle group with a single-block prompt stores it at the end of
prefill: prompt blocks are stable, so no tail is held back."""
prefill: prompt blocks are stable, so no tail is held back.
"""
block_size = 4
blocks_per_chunk = 1
tokens_per_chunk = block_size * blocks_per_chunk
@@ -4254,7 +4263,8 @@ def test_request_finished_mixed_full_attn_and_sliding_window(
def test_chunked_local_attention_reports_its_chunk_window():
"""Llama 4 uses chunked local attention, which used to trip the
FullAttentionSpec assert and kill the engine at startup."""
FullAttentionSpec assert and kill the engine at startup.
"""
spec = ChunkedLocalAttentionSpec(
block_size=16,
num_kv_heads=2,
@@ -4493,7 +4503,8 @@ def test_retention_interval_zero_stores_only_replay_boundary(
def _shared_kv_mtp_config():
"""Speculative config for a shared-group MTP model: eagle-family method
whose drafter layer merges into a target KV-cache group, so no group
self-identifies as a drafter group (issue #52735)."""
self-identifies as a drafter group (issue #52735).
"""
spec = MagicMock(name="shared_kv_mtp_spec")
spec.use_eagle.return_value = True
spec.use_eagle_block_drop.return_value = True
@@ -4507,12 +4518,14 @@ def _shared_kv_mtp_config():
class TestSharedGroupMTPOffload:
"""Regression tests for issue #52735: OffloadingConnector must keep
serving when speculative decoding is enabled but no KV-cache group is
annotated as a drafter group (shared-group MTP models)."""
annotated as a drafter group (shared-group MTP models).
"""
def test_no_annotation_marks_no_groups(self, request_runner):
"""Spec decode on + zero annotated groups must NOT mark every group
as a drafter group; the full store->load roundtrip must match the
non-speculative behavior of test_two_groups_full_and_sliding_window."""
non-speculative behavior of test_two_groups_full_and_sliding_window.
"""
block_size = 4
kv_cache_groups = [
KVCacheGroupSpec(
@@ -4569,7 +4582,8 @@ class TestSharedGroupMTPOffload:
class TestMambaHybridOffloadServing:
"""Store->finish->lookup flows on a full-attention + mamba-align hybrid,
with a manager that only HITs keys that were actually stored. Guards the
two collapse routes of issue #52735."""
two collapse routes of issue #52735.
"""
BLOCK = 4
MAMBA_BLOCK = 16
@@ -4698,7 +4712,8 @@ class TestMambaHybridOffloadServing:
"""Route 1 of #52735: with spec decode on and no drafter annotation,
serving must equal the non-speculative baseline (was 0 before the
fix: the all-groups fallback marked the mamba group as a drafter and
the volatile-tail pop consumed its only servable chunk)."""
the volatile-tail pop consumed its only servable chunk).
"""
baseline = self._roundtrip_served_tokens(self._make_scheduler(None))
assert baseline == 16
served = self._roundtrip_served_tokens(
@@ -4711,7 +4726,8 @@ class TestMambaHybridOffloadServing:
group must not drag the confirmed boundary below a coarser sibling's
chunk granularity. The widened query makes the volatile-tail pop
land on an extra queried chunk, holding the boundary at 16 tokens
(was 0 before the fix: 4 hits -> pop -> 12 tokens < mamba chunk)."""
(was 0 before the fix: 4 hits -> pop -> 12 tokens < mamba chunk).
"""
scheduler = self._make_scheduler(None, mamba_eagle=True)
assert [c.is_eagle_group for c in scheduler.config.kv_group_configs] == [
True,
@@ -80,7 +80,8 @@ def _allocate_kv_caches(
def _single_rank_vllm_config(total_kv_heads: int):
"""A one-rank (TP=1) parallel config, as canonical mappings are derived
from it."""
from it.
"""
vllm_config = MagicMock()
parallel_config = vllm_config.parallel_config
parallel_config.tensor_parallel_size = 1
@@ -98,9 +99,7 @@ def _make_worker(
replicated_layout: bool = False,
rank: int = 0,
):
"""
Create an OffloadingConnectorWorker with mocked dependencies.
"""
"""Create an OffloadingConnectorWorker with mocked dependencies."""
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.worker import (
OffloadingConnectorWorker,
)
@@ -475,8 +475,7 @@ class RequestRunner:
complete_transfers: bool,
post_step_fn: Callable[[], None] | None = None,
):
"""
Runs multiple engine (scheduler + worker) steps.
"""Runs multiple engine (scheduler + worker) steps.
Assumes a single request is running.
Args:
@@ -484,8 +483,8 @@ class RequestRunner:
complete_transfers: complete transfers immediately
post_step_fn: optional callback invoked after each step's
update_from_output(), before the next schedule().
"""
"""
tokens_iter = iter(decoded_tokens)
token_id = next(tokens_iter, None)
prev_scheduler_output = None
@@ -617,8 +616,7 @@ class RequestRunner:
expected_flushed: tuple[int | tuple[int, int], ...] = (),
post_step_fn: Callable[[], None] | None = None,
):
"""
Runs multiple engine (scheduler + worker) steps.
"""Runs multiple engine (scheduler + worker) steps.
Assumes a single request is running.
Args:
@@ -634,8 +632,8 @@ class RequestRunner:
A GPU block is either a (group_idx: int, request_block_offset: int)
or just request_block_offset: int.
The latter case is a convenience for representing all groups.
"""
"""
expected_stored_gpu_blocks = self._to_gpu_blocks(expected_stored)
expected_loaded_gpu_blocks = self._to_gpu_blocks(expected_loaded)
expected_flushed_gpu_blocks = self._to_gpu_blocks(expected_flushed)
@@ -148,7 +148,8 @@ def test_multiturn_lifecycle():
(do_remote_decode=True). Finishes LENGTH_CAPPED with remote_block_ids.
Turn 2: P receives remote_block_ids from D. P pulls KV from D because
remote_block_ids is not None and external tokens > 0. Computes only
new tokens, finishes LENGTH_CAPPED."""
new tokens, finishes LENGTH_CAPPED.
"""
vllm_config = create_vllm_config(
kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG,
)
@@ -194,7 +195,8 @@ def test_multiturn_lifecycle():
def test_first_turn_no_remote_blocks():
"""First turn: P has no remote_block_ids from D yet.
Standard local prefill, returns kv_transfer_params for future turns."""
Standard local prefill, returns kv_transfer_params for future turns.
"""
vllm_config = create_vllm_config(
kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG,
)
@@ -274,7 +276,8 @@ def test_abort_p_side_non_length_capped():
def test_remote_blocks_exceed_prompt_tokens():
"""D provides more remote tokens than P's prompt needs.
P caps external tokens to prompt length."""
P caps external tokens to prompt length.
"""
vllm_config = create_vllm_config(
kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG,
)
@@ -310,7 +313,8 @@ def test_remote_blocks_exceed_prompt_tokens():
def test_p_node_pulls_partial_last_block_from_d():
"""D sends remote_block_ids with partially filled last block.
remote_num_tokens < len(remote_block_ids) * block_size.
P pulls only remote_num_tokens worth of external tokens."""
P pulls only remote_num_tokens worth of external tokens.
"""
vllm_config = create_vllm_config(
kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG,
)
@@ -355,7 +359,8 @@ def test_p_node_pulls_partial_last_block_from_d():
def test_add_new_req_to_recv_populates_remote_meta():
"""add_new_req_to_recv correctly populates RemoteMeta for P-node
bi-directional KV pull from D."""
bi-directional KV pull from D.
"""
meta = NixlConnectorMetadata()
kv_params = {
"remote_block_ids": [[0, 1, 2]],
@@ -383,7 +388,8 @@ def test_add_new_req_to_recv_populates_remote_meta():
def test_build_connector_meta_recv_entries():
"""P-node scheduler: do_remote_decode=True + remote_block_ids →
_reqs_need_recv populated, build_connector_meta produces reqs_to_recv."""
_reqs_need_recv populated, build_connector_meta produces reqs_to_recv.
"""
vllm_config = create_vllm_config(
kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG,
)
@@ -457,7 +463,8 @@ def test_p_node_pull_kv_from_d(dist_init):
)
def test_p_node_pull_then_send_kv(dist_init):
"""Full P-node bi-directional: pull KV from D → prefill →
send KV back to D via notification."""
send KV back to D via notification.
"""
connector, worker = _make_connector_with_fake_worker()
meta = _make_p_node_recv_metadata("req-p2", [10, 11], [20, 21])
_do_load_kv(connector, meta)
@@ -504,7 +511,8 @@ def test_p_node_deferred_pull_on_no_handshake(dist_init):
def test_d_node_request_finished_returns_kv_params():
"""D-node request_finished returns kv_transfer_params with
do_remote_decode=True, remote_block_ids, remote_num_tokens
for P to pull. These params go directly to P node."""
for P to pull. These params go directly to P node.
"""
vllm_config = create_vllm_config(
kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG,
)
@@ -585,7 +593,8 @@ def test_d_node_request_finished_remote_num_tokens():
def test_d_node_partial_last_block_remote_num_tokens():
"""D-node: remote_num_tokens < len(remote_block_ids) * block_size
when last block is partially filled."""
when last block is partially filled.
"""
vllm_config = create_vllm_config(
kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG,
)
@@ -619,7 +628,8 @@ def test_no_double_read_blocks_after_reschedule():
bidirectional request (once on initial schedule, once after
WAITING_FOR_REMOTE_KVS → reschedule). The _remote_blocks_processed
flag must prevent the request from being added to _reqs_need_recv
twice, which would cause P to read D's blocks twice."""
twice, which would cause P to read D's blocks twice.
"""
vllm_config = create_vllm_config(
kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG,
)
@@ -672,7 +682,8 @@ def test_remote_num_tokens_bounded_by_blocks():
"""Edge case 2: D-node request_finished must return
remote_num_tokens <= len(remote_block_ids) * block_size.
request.num_tokens includes the last sampled token which has no KV
in the cache, so remote_num_tokens must use num_computed_tokens."""
in the cache, so remote_num_tokens must use num_computed_tokens.
"""
vllm_config = create_vllm_config(
kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG,
)
@@ -708,7 +719,8 @@ def test_remote_num_tokens_bounded_by_blocks():
def test_kv_recompute_threshold_skips_small_transfer():
"""Edge case 3: When remote tokens are below kv_recompute_threshold,
P should skip the remote pull and compute locally instead of
entering WAITING_FOR_REMOTE_KVS."""
entering WAITING_FOR_REMOTE_KVS.
"""
threshold = 256
vllm_config = create_vllm_config(
kv_connector_extra_config={
@@ -750,7 +762,8 @@ def test_p_node_finished_holds_blocks_for_d():
"""Edge case 4: P-node finishes with FINISHED_LENGTH_CAPPED and
do_remote_decode=True. P must hold blocks (delay_free=True) and
return kv_transfer_params with do_remote_prefill=True so D can
read P's blocks."""
read P's blocks.
"""
vllm_config = create_vllm_config(
kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG,
)
@@ -791,7 +804,8 @@ def test_p_node_finished_holds_blocks_for_d():
def test_cache_miss_first_turn_no_remote_pull():
"""Edge case 5: First turn with do_remote_decode=True but no
remote_block_ids (cache MISS). P should prefill locally with
num_external_tokens=0 and not enter WAITING_FOR_REMOTE_KVS."""
num_external_tokens=0 and not enter WAITING_FOR_REMOTE_KVS.
"""
vllm_config = create_vllm_config(
kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG,
)
@@ -826,7 +840,8 @@ def test_cache_miss_first_turn_no_remote_pull():
def test_partial_remote_tokens_less_than_prompt():
"""Edge case 6: D's remote_num_tokens covers only part of P's
prompt. P should pull remote_num_tokens worth of external tokens
and compute the rest locally."""
and compute the rest locally.
"""
vllm_config = create_vllm_config(
kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG,
)
@@ -871,7 +886,8 @@ def test_partial_remote_tokens_less_than_prompt():
def test_remote_blocks_processed_flag_persists():
"""Edge case 7: After recv completes and request is rescheduled,
the _remote_blocks_processed flag in kv_transfer_params prevents
the bidirectional path from re-entering _reqs_need_recv."""
the bidirectional path from re-entering _reqs_need_recv.
"""
vllm_config = create_vllm_config(
kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG,
)
@@ -943,7 +959,8 @@ _REMOTE = FakeNixlConnectorWorker.REMOTE_ENGINE_ID
)
def test_turn2_deadline_gate(dist_init, offset, expiry_delta, expect_declined):
"""P declines on the turn-2 readback if the deadline is known,
near-expiry and a handshake clock offset is known"""
near-expiry and a handshake clock offset is known
"""
connector, worker = _make_connector_with_fake_worker()
worker._engine_clock_offset[_REMOTE] = offset
expiry_time = None if expiry_delta is None else time.perf_counter() + expiry_delta
@@ -979,7 +996,8 @@ def test_turn2_deadline_gate(dist_init, offset, expiry_delta, expect_declined):
)
def test_turn2_full_prefix_hit_with_expired_deadline_skips_gate(dist_init):
"""A full local prefix hit issues no READ. The expiry gate must be skipped
even when D's deadline is expired."""
even when D's deadline is expired.
"""
connector, worker = _make_connector_with_fake_worker()
worker._engine_clock_offset[_REMOTE] = 0.0
@@ -1032,7 +1050,8 @@ def test_d_node_request_finished_exports_blocks_expiry_time():
def test_handshake_listener_appends_perf_counter_frame():
"""The handshake reply carries a 2nd frame with the listener's live
perf_counter."""
perf_counter.
"""
import threading
import msgspec
@@ -1,8 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
test that invalid blocks are evicted from prefix cache to prevent pollution.
"""test that invalid blocks are evicted from prefix cache to prevent pollution.
verifies that when sync-loading fails, invalid blocks are removed from the
prefix cache hash table so future requests cannot match and reuse corrupted data.
@@ -39,7 +38,7 @@ def _make_get_num_new_matched_tokens(
@pytest.fixture
def fail_scheduler():
"""scheduler with kv_load_failure_policy='fail'"""
"""Scheduler with kv_load_failure_policy='fail'."""
vllm_config = create_vllm_config()
vllm_config.kv_transfer_config.kv_load_failure_policy = "fail"
return create_scheduler(vllm_config)
@@ -48,8 +47,7 @@ def fail_scheduler():
def test_invalid_blocks_evicted_prevents_cache_pollution(
fail_scheduler: Scheduler,
):
"""
verify invalid blocks are evicted to prevent future cache hits.
"""Verify invalid blocks are evicted to prevent future cache hits.
scenario:
1. request 1 loads externally-computed blocks (sync mode)
+12 -6
View File
@@ -19,13 +19,15 @@ class _StubLMCacheMPConnector:
test only asserts on the connector *name* and the ``extra_config`` dict
produced by ``VllmConfig``, never instantiates the connector, so a bare
placeholder class is sufficient. Not subclassing ``SupportsHMA`` mirrors
the real connector's HMA support (it does not support HMA either)."""
the real connector's HMA support (it does not support HMA either).
"""
@pytest.fixture
def stub_lmcache_mp_connector(monkeypatch):
"""Replace the lazy loader so VllmConfig.__post_init__ does not import
``lmcache_mp_connector`` (and thus ``lmcache``) during config tests."""
``lmcache_mp_connector`` (and thus ``lmcache``) during config tests.
"""
monkeypatch.setitem(
KVConnectorFactory._registry,
"LMCacheMPConnector",
@@ -105,7 +107,8 @@ def _build_config(
enable_cumem_allocator: bool = False,
) -> VllmConfig:
"""Build a VllmConfig that exercises _verify_kv_transfer_compat without
requiring a real model (avoids HF downloads in CI)."""
requiring a real model (avoids HF downloads in CI).
"""
from types import SimpleNamespace
kv_transfer_config = (
@@ -131,7 +134,8 @@ def test_kv_connector_rejects_expandable_segments(monkeypatch, kv_connector):
invalidated when expandable_segments lets the CUDA VMM allocator remap
the underlying physical pages. We can't enumerate every connector that
does this (especially OOT ones), so reject the combination whenever any
connector is configured."""
connector is configured.
"""
monkeypatch.setenv("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
with pytest.raises(ValueError, match="expandable_segments"):
_build_config(kv_connector=kv_connector)
@@ -139,7 +143,8 @@ def test_kv_connector_rejects_expandable_segments(monkeypatch, kv_connector):
def test_kv_connector_allows_expandable_segments_with_sleep_mode(monkeypatch):
"""Sleep mode routes KV allocations through CuMemAllocator's pool, which
auto-disables expandable_segments (see #40812)."""
auto-disables expandable_segments (see #40812).
"""
monkeypatch.setenv("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
_build_config(kv_connector="NixlConnector", enable_sleep_mode=True)
@@ -162,7 +167,8 @@ def test_kv_connector_allows_other_alloc_conf(monkeypatch):
def test_no_kv_connector_ignores_expandable_segments(monkeypatch):
"""The expandable_segments check only applies when a KV connector is
configured."""
configured.
"""
monkeypatch.setenv("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
_build_config(kv_connector=None)

Some files were not shown because too many files have changed in this diff Show More