[CI] Ignore ruff D209, rejoin the docstrings it split, and silence incompatible-rule warnings (#57212)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-09-16 10:57:24 -07:00
committed by GitHub
co-authored by Claude Opus 5
parent 1085b64425
commit 651a88c09a
693 changed files with 2088 additions and 4171 deletions
@@ -1392,8 +1392,7 @@ def test_non_causal_backend_correctness(
model: str,
):
"""Test backend's correctness with non-causal (bidirectional) decoder
attention, as used by DFlash speculative decoding.
"""
attention, as used by DFlash speculative decoding."""
def bidirectional_mask_mod(
b: torch.Tensor,
@@ -115,8 +115,7 @@ def test_mamba_layers_have_unified_interface(
layer_class, expected_backend, expected_mamba_type
):
"""Test that all Mamba layers have the unified get_attn_backend
interface.
"""
interface."""
assert hasattr(layer_class, "get_attn_backend"), (
f"{layer_class.__name__} should have get_attn_backend method"
)
@@ -164,8 +164,7 @@ def apply_split_decodes_and_prefills(
treat_short_extends_as_decodes: bool = True,
):
"""Helper function to apply split_decodes_and_prefills and return
the results.
"""
the results."""
device = torch.device("cpu")
seq_lens = [10 * (i + 1) for i in range(len(query_lens))]
common_metadata = create_common_attn_metadata(
+1 -2
View File
@@ -1,8 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""The empty-shard mask fused into the DCP A2A pack kernel must be bit-exact
with the eager ``mask_dcp_empty_shards_`` pass it replaces.
"""
with the eager ``mask_dcp_empty_shards_`` pass it replaces."""
import pytest
import torch
@@ -495,8 +495,7 @@ def test_dspark_noncausal_sparse_mla_matches_sdpa(
num_heads,
):
"""Non-causal (window ∪ block, future-pointing) per-token indices must match
a dense SDPA reference over the same indices, for both sparse-MLA backends.
"""
a dense SDPA reference over the same indices, for both sparse-MLA backends."""
_skip_if_backend_unavailable(backend_cls, kv_cache_dtype, block_size)
window, block, topk_width = _DSPARK_CONFIGS[config_name]
@@ -231,8 +231,7 @@ def test_plan_uses_state_params(monkeypatch):
def test_kv_lens_host_formula():
"""Per-row host lengths: context == position + 1; capped at
index_topk + trailing-pool remainder past the sparse threshold.
"""
index_topk + trailing-pool remainder past the sparse threshold."""
builder = object.__new__(FlashInferMLASparseSM90Builder)
builder._index_topk = 2048
builder._index_kpool = 4
@@ -187,8 +187,7 @@ def test_gdn_build_classification(test_case: GDNBuildTestCase):
def test_has_initial_state_after_reclassification():
"""After reclassification, num_prefills > 0 so the prefill kernel path
should compute has_initial_state. For the reclassified request with
context_lens > 0, the corresponding entry must be True.
"""
context_lens > 0, the corresponding entry must be True."""
builder = _create_gdn_builder(num_speculative_tokens=2)
batch = BatchSpec(seq_lens=[65, 20], query_lens=[1, 3])
meta = _build(builder, batch, num_decode_draft_tokens=[-1, 2])
@@ -56,8 +56,7 @@ def _ref_stable_topk_from_candidates_fp64(
) -> torch.Tensor:
"""Pure-PyTorch reference for the CuteDSL stable-topk selector order
(score desc, then lowest global token id). Selects the same SET as the
kernel; only the set is compared in tests.
"""
kernel; only the set is compared in tests."""
num_rows, num_candidates = candidate_scores.shape
device = candidate_scores.device
select_k = min(k, num_candidates)
@@ -122,8 +121,7 @@ def _dcp_lse_merge(
class _FakeDCPGroup:
"""Single-process stand-in: ``all_gather`` returns the pre-built
concatenation of every rank's packed ``(score, global_id)`` candidates,
mirroring the one packed all-gather the merge issues.
"""
mirroring the one packed all-gather the merge issues."""
def __init__(self, gathered_packed: torch.Tensor) -> None:
self.gathered_packed = gathered_packed
@@ -209,8 +207,7 @@ def _merge_local_topks_global_with_fake_dcp(
"""Run ``_merge_dcp_topk_global`` for every rank against a faked
all-gather and return each rank's global-index result (all ranks should
agree). The fake pre-builds the packed candidate concatenation the merge's
single ``all_gather`` would return.
"""
single ``all_gather`` would return."""
# The merge is now CuteDSL-only (no PyTorch fallback), so it runs the real
# Triton pack + CuteDSL selector kernels even behind the faked all-gather.
if not current_platform.is_cuda() or not has_cutedsl():
@@ -712,8 +709,7 @@ def test_dcp_filter_compaction_matches_reference(
"""In-kernel compaction must, for every row, produce exactly the rank-owned
physical slots packed into [0, valid_count) with -1 in the tail -- the same
SET a reference filter + sort/gather produces. Uses wide rows (> BLOCK_N
valid slots), with interior -1 gaps.
"""
valid slots), with interior -1 gaps."""
device = torch.device("cuda")
torch.manual_seed(7)
dcp_size = 2
@@ -871,8 +867,7 @@ def test_decode_topk_pads_surplus_with_negative_one():
so a non-(-1) pad would be silently attended. This is the common case under
DCP, where each rank's local seq_len = global / world is usually << topk.
Checked on the *set* of valid indices (the kernel may order them by score,
not position).
"""
not position)."""
torch.manual_seed(0)
device = torch.device("cuda")
topk = 8
@@ -918,8 +913,7 @@ def test_sparse_decode_dcp_short_context_matches_non_dcp():
"""End-to-end DCP decode where the global seq_len < topk (so every rank's
local top-k is surplus-padded). Exercises the kernel surplus -> merge mask
-> global top-k -> physical localize -> LSE merge chain for the common
short-context decode case, vs the non-DCP reference.
"""
short-context decode case, vs the non-DCP reference."""
torch.manual_seed(4)
device = torch.device("cuda")
world = 2
@@ -58,8 +58,7 @@ def test_tail_backend_layout_matches_kernel_pointer_arithmetic():
def make_tail_block_table(own_blocks, width=64):
"""Tail-group block table as BlockTables produces it: column 0 holds the
request's single KpoolTailManager block, the remaining columns are never
written and stay zero.
"""
written and stay zero."""
bt = torch.zeros(len(own_blocks), width, dtype=torch.int32)
bt[:, 0] = torch.tensor(own_blocks, dtype=torch.int32)
return bt
@@ -67,8 +66,7 @@ def make_tail_block_table(own_blocks, width=64):
def legacy_generic_tail_slots(block_table, query_start_loc, positions):
"""Reference of the generic ``_compute_slot_mappings_kernel`` arithmetic
(block_table.py:305-313) applied to the tail group's table.
"""
(block_table.py:305-313) applied to the tail group's table."""
slots = []
for req in range(block_table.shape[0]):
for i in range(query_start_loc[req], query_start_loc[req + 1]):
@@ -109,8 +107,7 @@ def make_batch(per_req_positions, padded_len=None):
def test_legacy_generic_mapping_collapses_onto_block_zero():
"""The bug: with the manager's 1-column block table, the generic kernel
maps every pos >= kpool onto tail block 0, and distinct requests collide.
"""
maps every pos >= kpool onto tail block 0, and distinct requests collide."""
own_blocks = [5, 9]
per_req = [list(range(10)), list(range(12))] # prompts of len 10 and 12
positions, qsl, _, num_actual, num_reqs = make_batch(per_req)
@@ -140,8 +137,7 @@ def test_legacy_generic_mapping_collapses_onto_block_zero():
def test_circular_mapping_isolates_requests():
"""The fix: every token lands in its own request's block at pos % kpool,
and no slot is ever shared by two different requests (slots do recur
within a request every kpool positions -- that is the circular design).
"""
within a request every kpool positions -- that is the circular design)."""
own_blocks = [5, 9]
per_req = [list(range(10)), list(range(12))]
positions, qsl, slot_mapping, num_actual, num_reqs = make_batch(per_req)
@@ -169,8 +165,7 @@ def test_circular_mapping_isolates_requests():
def test_circular_mapping_matches_generic_for_short_requests(prompt_len):
"""For pos < kpool the generic kernel already picks the own block, so the
two mappings agree while every position fits the request's first block
(single-request behavior is unchanged).
"""
(single-request behavior is unchanged)."""
own_blocks = [7]
per_req = [list(range(prompt_len))]
positions, qsl, slot_mapping, num_actual, num_reqs = make_batch(per_req)
@@ -280,8 +275,7 @@ class TailRingMirror:
"""Mirror of _kpool_tail_seed_kernel / _kpool_decode_update_batched_kernel
addressing: block = tail_slot // kpool, ring offset = pos % kpool; a pool
completing at pos reads ring slots (pool_start + s) % kpool and uses the
current token's own K/score for the last member.
"""
current token's own K/score for the last member."""
def __init__(self, num_blocks, kpool=KPOOL):
self.kpool = kpool
@@ -324,8 +318,7 @@ def tail_slot_for(mapping, req, pos, own_block):
def run_scenario(mapping, interleave):
"""Requests A (block 5, prompt len 9) and B (block 9, prompt len 11)
decode concurrently; returns A's boundary pool [8, 9, 10, 11].
"""
decode concurrently; returns A's boundary pool [8, 9, 10, 11]."""
ring = TailRingMirror(num_blocks=16)
blocks = {"A": 5, "B": 9}
prompts = {"A": 9, "B": 11}
@@ -28,8 +28,7 @@ def _make_vllm_config(
block_size: int | None = None,
):
"""Create a minimal mock VllmConfig with only the fields the builder
accesses, avoiding any model download / HF config inspection.
"""
accesses, avoiding any model download / HF config inspection."""
speculative_config = (
SimpleNamespace(
num_speculative_tokens=num_speculative_tokens,
@@ -75,8 +74,7 @@ def test_mamba_single_token_prompt_runs_as_prefill():
def test_update_block_table_copies_block_idx_to_persistent_buffers():
"""update_block_table() must write block_idx tensors to the current
builder's persistent buffers, not leave them pointing to a different
builder's buffers.
"""
builder's buffers."""
block_size = 16
max_model_len = 256
num_reqs = 4
@@ -180,8 +178,7 @@ def test_state_indices_tensor_d_includes_num_speculative_blocks():
bug 1: with mamba_cache_mode='all' and speculative decoding enabled,
the cudagraph buffer for state_indices_tensor_d must allocate the same
per-request column count as the runtime block table, which includes
num_speculative_blocks trailing scratch columns.
"""
num_speculative_blocks trailing scratch columns."""
block_size = 16
max_model_len = 256
max_num_seqs = 4
@@ -214,8 +211,7 @@ def test_block_idx_cudagraph_capture_padded_by_num_reqs():
bug 2: with mamba_cache_mode='all' and spec decode, _update_metadata_for
_cudagraph_capture must slice block_idx_last_{scheduled,computed}_token
by the request count (padded_bs == num_reqs), not by num_decode_tokens.
Past num_decodes, the slice must be zero-filled.
"""
Past num_decodes, the slice must be zero-filled."""
block_size = 16
max_model_len = 256
max_num_seqs = 8
@@ -295,8 +291,7 @@ def test_block_idx_prev_step_persistent_buffer_allocated():
block_idx_last_scheduled_token_prev_step as a persistent buffer with the
same shape as the existing block_idx_last_{scheduled,computed}_token
buffers, so cudagraph capture records a stable pointer for the prev-step
input anchor consumed by mamba_mixer2's input gather.
"""
input anchor consumed by mamba_mixer2's input gather."""
block_size = 16
max_model_len = 256
max_num_seqs = 8
@@ -324,8 +319,7 @@ def test_block_idx_prev_step_persistent_buffer_allocated():
def test_block_idx_prev_step_persistent_buffer_skipped_without_spec_decode():
"""Without spec decode, the prev-step buffer is unused and must not be
allocated — the input anchor reduces to last_computed_token.
"""
allocated — the input anchor reduces to last_computed_token."""
block_size = 16
max_model_len = 256
max_num_seqs = 8
@@ -349,8 +343,7 @@ def test_block_idx_prev_step_cudagraph_capture_uses_persistent_buffer():
"""_update_metadata_for_cudagraph_capture must copy the prev-step anchor
into the builder's persistent buffer (so cudagraph replay reads from the
same underlying memory), pad past num_decodes with zero, and return a
slice of the persistent buffer in the metadata.
"""
slice of the persistent buffer in the metadata."""
block_size = 16
max_model_len = 256
max_num_seqs = 8
+1 -2
View File
@@ -171,8 +171,7 @@ def test_glm5_flashinfer_masked_mha_routing(
def test_concat_k_nope_k_pe_matches_torch_cat(qk_rope_head_dim):
"""The K concat used by the MLA prefill context loop must equal torch.cat of
k_nope with the broadcast k_pe; with no RoPE part it returns k_nope itself
instead of allocating and copying.
"""
instead of allocating and copying."""
torch.manual_seed(0)
num_tokens, num_heads, qk_nope_head_dim = 5, 4, 256
k_nope = torch.randn(num_tokens, num_heads, qk_nope_head_dim, dtype=torch.bfloat16)
@@ -134,8 +134,7 @@ def test_flash_attn_registers_warmup_only_for_fa4(
def test_flash_attn_prefill_backend_signature_accepts_fused_kwargs():
"""run_prefill_new_tokens must accept out/output_scale so the direct
(non-**kwargs) call in forward_mha type- and runtime-checks.
"""
(non-**kwargs) call in forward_mha type- and runtime-checks."""
import inspect
params = inspect.signature(
@@ -151,8 +150,7 @@ def test_flash_attn_prefill_backend_signature_accepts_fused_kwargs():
def test_mla_impl_forward_mha_accepts_output_scale():
"""The abstract MLA impl forward_mha must carry output_scale so every
override (and the unconditional forward_impl call) stays compatible.
"""
override (and the unconditional forward_impl call) stays compatible."""
import inspect
from vllm.v1.attention.backend import MLAAttentionImpl
@@ -179,8 +177,7 @@ _FUSED_FP8_SKIP = _fused_fp8_skip_reason()
def test_fa4_fused_fp8_output_matches_post_quant(default_vllm_config):
"""FA4's fused FP8 write (output_scale, flash-attention#135) must match the
bf16-attention + standalone static-FP8-quant path it replaces, since
production uses the same output_scale for both.
"""
production uses the same output_scale for both."""
from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
from vllm.platforms import current_platform
@@ -315,8 +315,7 @@ class TestBackendValidation:
def test_flash_attn_accepts_glm53_flash_nope_dimensions(self):
"""(256, 0, 256) runs the same FA kernels as GLM-5's (192, 64, 256); a
RoPE-carrying 320-wide query does not.
"""
RoPE-carrying 320-wide query does not."""
try:
from vllm.v1.attention.backends.mla.prefill.flash_attn import (
FlashAttnPrefillBackend,
@@ -257,8 +257,7 @@ def test_replayssm_write_pos(case: ReplaySSMBuildCase):
def test_resumed_request_differs_from_fresh():
"""Same token count, different decode_base: fresh (base 100) -> write_pos 5,
resumed (base 105) -> write_pos 0.
"""
resumed (base 105) -> write_pos 0."""
builder = _create_replayssm_builder(16)
batch = BatchSpec(seq_lens=[106, 106], query_lens=[1, 1])
common = create_common_attn_metadata(batch, BLOCK_SIZE, DEVICE).replace(
@@ -517,8 +517,7 @@ def test_auto_selection_for_kv_connector(
aiter_found, expected_backend, mock_vllm_config, mock_get_cdna_version
):
"""Auto-selection with a KV connector and AITER enabled resolves to unified attn,
and to triton attn if AITER not enabled.
"""
and to triton attn if AITER not enabled."""
from vllm.platforms.rocm import RocmPlatform
with patch(
@@ -102,8 +102,7 @@ def expected_row_seq_lens(per_req_positions, batch_size, next_n):
def test_legacy_flat_layout_misaligns_and_bleeds():
"""The bug: ``positions[:n] + 1`` with padded-row coordinates reads other
requests' positions and prefill positions.
"""
requests' positions and prefill positions."""
decode_lens, per_req, positions = make_non_uniform_batch()
batch_size = decode_lens.shape[0]
next_n = int(decode_lens.max())
@@ -138,8 +137,7 @@ def test_helper_uniform_layout_matches_flat_slice():
def test_helper_padded_layout_per_row():
"""Non-uniform batches map every padded row to its own token's position;
pad rows get 0 (empty tail).
"""
pad rows get 0 (empty tail)."""
decode_lens, per_req, positions = make_non_uniform_batch()
out = _decode_topk_seq_lens(positions, decode_lens, 8, 3, 4, requires_padding=True)
expected = expected_row_seq_lens(per_req, 3, 4)
@@ -151,8 +149,7 @@ def test_helper_padded_layout_per_row():
def expand_tail_region(dec_seq):
"""Run the production pure-torch expand + append pair (the fused kernel
replicates it exactly on the identity path) and return the tail columns
[TOPK_TOKENS, TOPK_TOKENS + KPOOL - 1).
"""
[TOPK_TOKENS, TOPK_TOKENS + KPOOL - 1)."""
rows = dec_seq.shape[0]
pool_ids = torch.arange(SELECT_K, dtype=torch.int64).expand(rows, SELECT_K)
valid = torch.ones_like(pool_ids, dtype=torch.bool)
@@ -166,8 +163,7 @@ def expand_tail_region(dec_seq):
def test_tail_expansion_legacy_vs_fixed():
"""End-to-end tail consequence: the fixed mapping appends exactly the
request's trailing incomplete pool; the legacy flat slice drops real tail
tokens and emits indices far past the request's own sequence.
"""
tokens and emits indices far past the request's own sequence."""
decode_lens, per_req, positions = make_non_uniform_batch()
n = 12
fixed = _decode_topk_seq_lens(
+9 -18
View File
@@ -909,8 +909,7 @@ def test_index_group_convert_during_piecewise_capture():
runs in the following eager break. The convert's side stream must not
wait on ``logical_topk_ready`` there: an event recorded inside a captured
segment is graph-local, and an eager wait on it raises
cudaErrorInvalidValue.
"""
cudaErrorInvalidValue."""
device = torch.device(DEVICE_TYPE)
num_tokens, num_topk, num_requests, blocks_per_req, block_size = 8, 128, 4, 4, 16
@@ -1049,8 +1048,7 @@ def test_triton_convert_rejects_req_id_longer_than_token_indices():
req_id but the output is allocated like token_indices, so a full-batch
req_id combined with an MQA-subset token_indices wrote past the end of
the output buffer. The wrapper must reject the length mismatch instead
of corrupting memory.
"""
of corrupting memory."""
device = torch.device(DEVICE_TYPE)
num_topk_tokens = 128
block_size = 64
@@ -1105,8 +1103,7 @@ def test_flashmla_forward_bf16_kv_slices_req_id_to_mqa_tokens():
is active, forward_mqa only receives the leading decode tokens, but
_forward_bf16_kv passed the full-batch req_id_per_token to the index
conversion, making it write past the end of its output buffer. The call
site must slice req_id_per_token to the MQA tokens.
"""
site must slice req_id_per_token to the MQA tokens."""
device = torch.device(DEVICE_TYPE)
num_topk_tokens = 128
block_size = 64
@@ -1204,8 +1201,7 @@ def test_split_prefill_chunks(seq_lens, max_buf, expected):
)
def test_masked_mha_workspace_fits_single_request_boundary(max_query_len, expected):
"""A 32K prefill needs the default workspace exactly; shrinking it would
push a supported request onto MQA.
"""
push a supported request onto MQA."""
assert (
_masked_mha_workspace_fits(
batch_size=1,
@@ -1263,8 +1259,7 @@ def test_is_masked_mha_available_model_dims(
):
"""The allow-list gates masked MHA per exact model geometry: the DeepSeek-V3.2,
GLM-5 and NoPE GLM-5.3-Flash layouts on an SM100-family GPU with FA4 and an
unquantized KV cache, nothing else.
"""
unquantized KV cache, nothing else."""
import vllm.model_executor.layers.attention.sparse_mla_attention as mod
monkeypatch.setattr(
@@ -3604,8 +3599,7 @@ def test_fp8_mixed_batch_dcp_neutralizes_empty_rows(monkeypatch):
"""A decode row whose top-k shard holds no local candidates (all -1) has
undefined kernel out/lse; it must come back as (0, -inf), the identity of
the cross-rank LSE merge, or a NaN would survive the merge even at zero
weight (0 * NaN = NaN).
"""
weight (0 * NaN = NaN)."""
num_tokens, num_heads, head_dim = 3, 2, 3
q = torch.empty(num_tokens, num_heads, head_dim, device=DEVICE_TYPE)
local_indices = torch.tensor(
@@ -3778,8 +3772,7 @@ def test_hisparse_fp8_prefill_gather_uses_dedicated_stream(monkeypatch):
def test_sparse_impl_observes_repointed_indexer_buffer():
"""The MTP proposer repoints the draft's indexer at the target model's buffer
after the backend impl is built, so the impl must resolve the buffer per read.
Snapshotting it in __init__ leaves the layer reading indices nothing writes.
"""
Snapshotting it in __init__ leaves the layer reading indices nothing writes."""
impl = object.__new__(FlashInferMLASparseImpl)
own = torch.zeros(4, 8, dtype=torch.int32)
target = torch.ones(4, 8, dtype=torch.int32)
@@ -3795,8 +3788,7 @@ def test_sparse_impl_observes_repointed_indexer_buffer():
def test_explicit_topk_buffer_supersedes_indexer():
"""Backbone skip-topk layers have no indexer, and the proposer also assigns the
shared buffer directly onto draft submodules.
"""
shared buffer directly onto draft submodules."""
impl = object.__new__(FlashInferMLASparseImpl)
indexer = SimpleNamespace(
topk_indices_buffer=torch.zeros(2, 2, dtype=torch.int32),
@@ -3811,8 +3803,7 @@ def test_explicit_topk_buffer_supersedes_indexer():
def test_sparse_mla_common_impl_resolves_buffer_lazily():
"""Guards the whole SparseMLACommonImpl family at once: a backend that
snapshots the buffer instead silently loses MTP buffer sharing.
"""
snapshots the buffer instead silently loses MTP buffer sharing."""
assert issubclass(SparseMLACommonImpl, SharedTopkIndicesBuffer)
assert isinstance(SparseMLACommonImpl.topk_indices_buffer, property), (
"topk_indices_buffer must stay a lazily-resolved property"
@@ -198,8 +198,7 @@ def _create_nvfp4_hnd_kv_cache(
def _run_trtllm_integration(batch_spec, kv_cache_dtype="auto", model_name=MODEL):
"""Run TRTLLM attention through the full FlashInfer pipeline
and compare against an SDPA reference.
"""
and compare against an SDPA reference."""
set_random_seed(42)
device = torch.device(f"{DEVICE_TYPE}:0")
@@ -427,8 +426,7 @@ def _run_trtllm_integration(batch_spec, kv_cache_dtype="auto", model_name=MODEL)
def test_trtllm_gen_full_attention_integration(batch_spec_name: str):
"""Test TRTLLM gen-full attention through the full FlashInfer
MetadataBuilder.build() -> FlashInferImpl.forward() pipeline,
with real TRTLLM kernels on Blackwell.
"""
with real TRTLLM kernels on Blackwell."""
_run_trtllm_integration(BATCH_SPECS[batch_spec_name])
@@ -439,8 +437,7 @@ def test_trtllm_gen_full_attention_integration(batch_spec_name: str):
@torch.inference_mode()
def test_trtllm_gen_nvfp4_kv_integration(batch_spec_name: str):
"""Test TRTLLM attention with nvfp4 KV cache through the full
FlashInfer MetadataBuilder.build() -> FlashInferImpl.forward() pipeline.
"""
FlashInfer MetadataBuilder.build() -> FlashInferImpl.forward() pipeline."""
_run_trtllm_integration(
BATCH_SPECS[batch_spec_name],
kv_cache_dtype="nvfp4",
@@ -2,8 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Fine-grained partial prefix-cache hits for hybrid (full attention + mamba
"align") models: scheduler chunk splitting, partial tail registration, CoW
on partial hits, and same-step deferral.
"""
on partial hits, and same-step deferral."""
from math import lcm
from types import SimpleNamespace
@@ -175,8 +174,7 @@ def test_mamba_align_split_partial_tail_schedule(dcp_world_size: int):
"""Chunk ends with partial hits on: block-aligned chunks, one extra stop
at the prompt's last hash boundary (registering the partial tail), then
the remaining tokens. block=512, hash=32, prompt=10000, budget=8192:
0 -> 8192 -> 9728 -> 9984 -> 10000.
"""
0 -> 8192 -> 9728 -> 9984 -> 10000."""
block_size = 512
scheduler_block_size = block_size * dcp_world_size
hash_block_size = 32
@@ -744,8 +742,7 @@ def test_hash_aligned_query_end_uses_regular_partial_tail():
def test_external_mamba_hit_same_block_uses_running_cow_on_continue():
"""An external mid-block hit must become a running request even when its
first continuation does not need another Mamba block.
"""
first continuation does not need another Mamba block."""
hash_block_size = 2
mamba_block_size = 4 * hash_block_size
kv_cache_config = KVCacheConfig(
@@ -817,8 +814,7 @@ def test_external_mamba_hit_same_block_uses_running_cow_on_continue():
def test_boundary_state_offloads_returns_cow_target():
"""Boundary hand-offs expose aligned snapshots and the partial-tail CoW
target, never the overwritten CoW source.
"""
target, never the overwritten CoW source."""
hash_block_size = 2
block_size = 2 * hash_block_size
kv_cache_config = KVCacheConfig(
@@ -1004,8 +1000,7 @@ def test_connector_finish_registers_partial_tail_before_cleanup():
def test_boundary_state_offload_dropped_when_request_freed_before_drain():
"""A hand-off recorded in the same scheduling pass as the request's death
must not be drained: its release hook has already run, so draining would
leak a pinned block.
"""
leak a pinned block."""
hash_block_size = 2
block_size = 2 * hash_block_size
kv_cache_config = KVCacheConfig(
@@ -1052,8 +1047,7 @@ def test_boundary_state_offload_dropped_when_request_freed_before_drain():
def test_boundary_state_offloads_block_aligned_prompt():
"""A prompt ending on a block boundary registers no CoW partial tail; its
boundary state block is handed off as a snapshot instead (once).
"""
boundary state block is handed off as a snapshot instead (once)."""
hash_block_size = 2
block_size = 2 * hash_block_size
kv_cache_config = KVCacheConfig(
@@ -1107,8 +1101,7 @@ def test_boundary_state_offloads_block_aligned_prompt():
def test_truncate_computed_blocks_preserves_sparse_prefix_positions():
"""truncate_computed_blocks slices each group by its own block size,
keeps null placeholders in the retained prefix, and leaves the original
lookup result untouched (pure view, no refcount changes).
"""
lookup result untouched (pure view, no refcount changes)."""
hash_block_size = 2
kv_cache_config = KVCacheConfig(
num_blocks=24,
@@ -1163,8 +1156,7 @@ def test_truncate_computed_blocks_preserves_sparse_prefix_positions():
def test_truncate_computed_blocks_allows_short_mamba_group_only():
"""External state may replace a short Mamba hit, but other groups must
cover the aligned local endpoint.
"""
cover the aligned local endpoint."""
hash_block_size = 2
kv_cache_config = KVCacheConfig(
num_blocks=24,
@@ -1306,8 +1298,7 @@ def test_hybrid_mamba_partial_tail_owner_continue_preserves_later_hit():
def test_hybrid_mamba_moved_partial_entry_defers_same_step_hit():
"""The owner's move re-arms the same-step guard: the moved entry is
filled by this step's copy, and chained same-step copies read stale
sources, so a request hitting it in the move step must be deferred.
"""
sources, so a request hitting it in the move step must be deferred."""
hash_block_size = 2
block_size = 2 * hash_block_size
kv_cache_config = KVCacheConfig(
@@ -1597,8 +1588,7 @@ def test_hybrid_partial_hash_truncates_full_attention_hit_length():
def test_cow_retained_blocks_returned_for_release():
"""new_step_starts returns the CoW copy retentions instead of freeing
them; the scheduler owns releasing them once the copy has run.
"""
them; the scheduler owns releasing them once the copy has run."""
hash_block_size = 2
block_size = 2 * hash_block_size
kv_cache_config = KVCacheConfig(
@@ -1651,8 +1641,7 @@ def test_cow_retained_blocks_returned_for_release():
def test_free_cow_retained_blocks_defers_until_copy_step_processed():
"""Scheduler releases CoW retentions immediately when the copy's step has
been processed (or deferral is off), and defers them otherwise.
"""
been processed (or deferral is off), and defers them otherwise."""
from collections import deque
freed: list = []
@@ -1689,8 +1678,7 @@ def test_free_cow_retained_blocks_defers_until_copy_step_processed():
def test_full_attention_eagle_drops_one_hash_unit():
"""With fine-grained partial hits, eagle rewinds the hit by one hash unit
instead of a whole cache block: the tail block's KV is append-only, so it
still covers the reduced length and stays in the hit as a partial block.
"""
still covers the reduced length and stays in the hit as a partial block."""
from vllm.v1.core.block_pool import BlockPool
from vllm.v1.core.single_type_kv_cache_manager import FullAttentionManager
@@ -1766,8 +1754,7 @@ def test_hybrid_partial_hit_with_eagle_stays_within_group_blocks():
"""Regression: with eagle, the mamba group must not receive the eagle
lookup margin — its finder never applies the drop, so it could return a
hit past the blocks the (dropped) full-attention group covers, crashing
the consumer's CoW with block_idx >= len(req_blocks).
"""
the consumer's CoW with block_idx >= len(req_blocks)."""
hash_block_size = 2
block_size = 2 * hash_block_size
kv_cache_config = KVCacheConfig(
@@ -2369,8 +2356,7 @@ def test_retention_snapshots_handed_off_with_exact_block_ids():
"""Under sparse retention, each retained mamba boundary state block is
handed to the connector with its exact block id. Connectors must not resolve
align-mode state blocks positionally because the block table is not
append-only.
"""
append-only."""
hash_block_size = 2
block_size = 4
manager = make_kv_cache_manager(
@@ -2411,8 +2397,7 @@ def test_snapshot_handoff_dense_default_retention():
"""With dense (default) retention, every materialized mamba boundary
state block is handed off — regular mamba-align + prefix-match-unit
deployments get store-able boundary snapshots without setting a
prefix-cache retention interval.
"""
prefix-cache retention interval."""
hash_block_size = 2
block_size = 4
manager = make_kv_cache_manager(
@@ -2444,8 +2429,7 @@ def test_snapshot_handoff_dense_default_retention():
def _run_chunked_prefill(manager, req, chunk_size, num_chunks):
"""Prefill ``req`` one ``chunk_size`` chunk per scheduler step, yielding the
boundary-state hand-offs offered (and claimed) at each step.
"""
boundary-state hand-offs offered (and claimed) at each step."""
computed_blocks, num_computed, _ = manager.get_computed_blocks(req)
for step in range(num_chunks):
manager.new_step_starts()
@@ -2509,8 +2493,7 @@ def test_boundary_states_offered_past_prompt_for_resumed_prefill():
``num_prompt_tokens``. A resumed request re-prefills its generated tokens
and every group re-saves them, so filtering on the original prompt length
would silently strip the mamba key for boundaries full attention still
stores; only the connector knows where its save window ends.
"""
stores; only the connector knows where its save window ends."""
block_size = 8
prompt_len = block_size
manager = make_kv_cache_manager(
+2 -4
View File
@@ -552,8 +552,7 @@ def _create_async_pp_scheduler(
def _assert_ordered_subset(delivered: list[int], emitted: list[int]) -> None:
"""Delivered tokens must be an order-preserving subset of the emitted
tokens with no duplicates (tokens are globally unique).
"""
tokens with no duplicates (tokens are globally unique)."""
it = iter(emitted)
for token in delivered:
assert token in it, f"token {token} delivered out of order or twice"
@@ -563,8 +562,7 @@ def _assert_positions_consistent(req, engine: PipelinedEngine) -> None:
"""The i-th delivered output token must be one the runner sampled for
exactly sequence position prompt_len + i: catches a preempted request's
stale output landing on a position the resumed request resampled (or
vice versa), which token-stream equality alone cannot see.
"""
vice versa), which token-stream equality alone cannot see."""
for i, token in enumerate(req.output_token_ids):
expected = req.num_prompt_tokens + i
actual = engine.emitted_position[token]
+6 -12
View File
@@ -178,8 +178,7 @@ def _shared_layout_config():
class TestCSALinearGrouping:
"""A CSA + linear-attention model (sparse attention with a compressor ring,
plus sharded GDN and one TP-replicated PLE state) goes through the generic
packed-group path; no model-specific branch is involved.
"""
packed-group path; no model-specific branch is involved."""
@staticmethod
def _mamba_groups(groups):
@@ -194,8 +193,7 @@ class TestCSALinearGrouping:
def test_replicated_state_gets_its_own_group(self):
"""The PLE state is TP-replicated and a different size, so it must not
share a manager group with the sharded GDN states (the NIXL worker
requires exactly one single-layer replicated group).
"""
requires exactly one single-layer replicated group)."""
groups = get_kv_cache_groups(_shared_layout_config(), _make_csa_linear_specs())
mamba_groups = self._mamba_groups(groups)
@@ -248,8 +246,7 @@ class TestCSALinearGrouping:
def test_scratch_group_survives_computed_block_truncation(self):
"""The scratch group contributes no computed blocks, so truncating a
lookup result must skip it: its block size is the ring capacity, which
neither divides the hit length nor bounds the (empty) block list.
"""
neither divides the hit length nor bounds the (empty) block list."""
config = _shared_layout_config()
config.cache_config.enable_prefix_caching = True
groups = get_kv_cache_groups(config, _make_csa_linear_specs())
@@ -363,8 +360,7 @@ class TestCSALinearGrouping:
per page size, which is emitted whole, or a single-layer state that
cannot be split at all -- the mamba layers must be sized against it.
Sizing them against a narrower bucket splits them past what the block
already fits, spending a pool block per extra group for no saving.
"""
already fits, spending a pool block per extra group for no saving."""
config = _mock_vllm_config("BLNHC")
config.speculative_config = None
specs = {}
@@ -409,8 +405,7 @@ class TestSlidingWindowBucketCap:
unbalanced 8-layer paged MLA bucket. Left whole, the SWA bucket would
set the block width and the MLA group would fill under a third of
every block; capped, it splits into groups no wider than the MLA
page.
"""
page."""
config = _shared_layout_config()
specs: dict[str, KVCacheSpec] = {}
for layer, ratio in ((2, 2), (8, 2), (14, 2), (20, 1)):
@@ -611,8 +606,7 @@ class TestCompressorRingGroup:
"""DeepSeek-V4.1 compressor state: a CircularBufferSpec ring (capacity
8) beside 128-token paged MLA groups and 64-token SWA groups. The ring
claims one block per request, never hashes, and does not disturb the
scheduler/hash block sizes the paged groups imply.
"""
scheduler/hash block sizes the paged groups imply."""
config = _shared_layout_config()
config.cache_config.enable_prefix_caching = True
specs: dict[str, KVCacheSpec] = {
+3 -6
View File
@@ -367,8 +367,7 @@ def test_free_request_with_duplicate_mm_hashes():
"""Freeing a request whose two inputs share the same mm_hash must fully
clean up request_cached_ids. After the first free_encoder_input call,
cached[mm_hash] becomes empty; the second call must still remove the
remaining input_id from request_cached_ids.
"""
remaining input_id from request_cached_ids."""
manager = EncoderCacheManager(cache_size=20)
req = MockRequest("r1", ["imgA", "imgA"], [4, 4])
@@ -384,8 +383,7 @@ def test_free_request_with_duplicate_mm_hashes():
def test_duplicate_mm_hash_stays_referenced_until_last_free():
"""`cached` holds one reference per request, not per position, so freeing
the first of two occurrences must not release the entry.
"""
the first of two occurrences must not release the entry."""
manager = EncoderCacheManager(cache_size=20)
req = MockRequest("r1", ["imgA", "imgA"], [4, 4])
@@ -406,8 +404,7 @@ def test_duplicate_mm_hash_stays_referenced_until_last_free():
def test_duplicate_mm_hash_is_not_evicted_before_last_use():
"""An item repeated within one request (an image carried across
conversation turns) must not become reclaimable capacity while a later
occurrence still has to be spliced in, or the encoder recomputes it.
"""
occurrence still has to be spliced in, or the encoder recomputes it."""
manager = EncoderCacheManager(cache_size=8)
req = MockRequest("r1", ["imgA", "imgA"], [4, 4])
other = MockRequest("r2", ["imgB"], [8])
+10 -20
View File
@@ -2593,8 +2593,7 @@ def test_get_kv_cache_config_kpool_tail_coowns_indexer_tensor():
"""The kpool tail parasitizes the indexer tensors instead of getting its
own: sibling idx/tail tensors paired by layer order, zero standalone tail
bytes, one shared block per request, and no prefix-caching leakage from
the kpool-sized scratch group.
"""
the kpool-sized scratch group."""
model_config = ModelConfig(max_model_len=8192)
vllm_config = VllmConfig(model_config=model_config)
@@ -2665,8 +2664,7 @@ def test_get_kv_cache_config_kpool_tail_coowns_indexer_tensor():
def test_glm5_kpool_tail_does_not_drag_hash_block_size():
"""The tail's kpool-sized scratch block (4 tokens) must not constrain the
prefix-cache hash granularity: participating groups alone decide it.
"""
prefix-cache hash granularity: participating groups alone decide it."""
model_config = ModelConfig(max_model_len=8192)
vllm_config = VllmConfig(model_config=model_config)
@@ -2742,8 +2740,7 @@ def test_get_kv_cache_config_mamba_hybrid_sharing_infeasible_no_indexer():
def test_get_kv_cache_config_mamba_hybrid_sharing_prepadded_mamba():
"""Platform-prepadded mamba pages (mamba_page_size_padded hint) must not
disable slot sharing; the layout re-pads them to the MLA page.
"""
disable slot sharing; the layout re-pads them to the MLA page."""
model_config = ModelConfig(max_model_len=8192)
vllm_config = VllmConfig(model_config=model_config)
@@ -2774,8 +2771,7 @@ def test_get_kv_cache_config_mamba_hybrid_sharing_prepadded_mamba():
def test_get_kv_cache_config_mamba_hybrid_sharing_pp_balanced_projection():
"""Round-robin mamba grouping keeps practical PP splits balanced: every
stage's largest projected mamba group slice fits its projected MLA
layers, so per-stage slot tensors all have an MLA owner.
"""
layers, so per-stage slot tensors all have an MLA owner."""
model_config = ModelConfig(max_model_len=8192)
vllm_config = VllmConfig(model_config=model_config)
@@ -2819,8 +2815,7 @@ def test_get_kv_cache_config_mamba_hybrid_sharing_pp_group_count_bump(monkeypatc
"""PP=4's default partition [11,11,12,11] leaves stage 0 with 2 MLA
layers but a round-robin slice of 3 under the minimum 4 mamba groups;
grouping bumps to 5 groups so every stage's projection keeps sharing
on instead of silently falling back on stage 0.
"""
on instead of silently falling back on stage 0."""
from vllm.config import ParallelConfig
from vllm.distributed.utils import get_pp_indices
@@ -2869,8 +2864,7 @@ def test_get_kv_cache_config_mamba_hybrid_sharing_pp_starved_stage(monkeypatch):
def test_get_kv_cache_config_mamba_hybrid_sharing_beats_cross_layers_flag():
"""Hybrid slot sharing must take precedence over the experimental
enable_cross_layers_blocks packed layout: generic packing would give the
MLA slots a strided view, breaking the contiguous virtual split.
"""
MLA slots a strided view, breaking the contiguous virtual split."""
model_config = ModelConfig(max_model_len=8192)
kv_transfer_config = KVTransferConfig(
kv_connector="NixlConnector",
@@ -3172,8 +3166,7 @@ def test_multi_run_layer_compact_strides_place_hoisted_heads():
"""A layer-compact run region is its own dense allocation: under LHBNC the head
groups sit between the layers and the blocks, so a run's block stride is one head
group's slice, not the whole page (regression test for setStorage out-of-bounds
on ROCm hybrid models).
"""
on ROCm hybrid models)."""
full = new_kv_cache_spec()
swa = new_sliding_window_spec(sliding_window=full.block_size * 2)
page = full.page_size_bytes
@@ -3252,8 +3245,7 @@ def test_mla_with_incompatible_swa_uses_one_full_allocation_group(caplog_vllm):
def test_hidden_states_with_tp_scales_page_size():
"""When TP shrinks KV pages below the hidden-state per-token cost,
get_kv_cache_groups must scale up target block sizes so that the
common page accommodates the unsharded hidden states.
"""
common page accommodates the unsharded hidden states."""
# Simulate TP=4 sharding a model with 8 KV heads → 2 per rank.
# KV page = block_size(16) * num_kv_heads(2) * head_size(64) * dtype(2)
# = 16 * 2 * 64 * 2 = 4096 bytes.
@@ -4083,8 +4075,7 @@ def test_check_enough_kv_cache_memory_reserves_null_block():
def test_is_full_attention_spec_unwraps_uniform_type_specs():
"""``UniformTypeKVCacheSpecs`` is not a ``FullAttentionSpec``, so callers
scanning groups with a bare isinstance miss DeepSeek-V4-shaped configs.
"""
scanning groups with a bare isinstance miss DeepSeek-V4-shaped configs."""
common = dict(num_kv_heads=1, head_size=1, dtype=torch.float32)
full = FullAttentionSpec(block_size=4, **common)
mla = MLAAttentionSpec(block_size=4, **common)
@@ -4253,8 +4244,7 @@ def test_no_warning_when_draft_group_is_identified(caplog_vllm):
def _deepseek_v4_specs(model_version="deepseek_v4"):
"""DeepseekV4-shaped specs: full MLA layers plus sliding-window MLA layers
at differing page sizes, with the MTP draft layer registered last.
"""
at differing page sizes, with the MTP draft layer registered last."""
return {
"model.layers.0.self_attn.attn": new_mla_spec(),
"model.layers.1.self_attn.attn": new_mla_spec(),
+3 -6
View File
@@ -55,8 +55,7 @@ def _mm_feature(offset: int, length: int) -> MultiModalFeatureSpec:
def test_strip_covered_mm_data() -> None:
"""Items fully inside the computed prefix lose their data; items touching
the uncomputed region keep it; already-None data stays None.
"""
the uncomputed region keep it; already-None data stays None."""
from dataclasses import replace
covered = _mm_feature(offset=0, length=100)
@@ -108,8 +107,7 @@ def _mm_feature_mixed(offset: int, length: int) -> MultiModalFeatureSpec:
def test_strip_covered_mm_data_mrope() -> None:
"""For M-RoPE models, covered items keep their keep_on_cpu metadata fields
(the worker needs them to compute positions); payload fields are dropped.
"""
(the worker needs them to compute positions); payload fields are dropped."""
covered = _mm_feature_mixed(offset=0, length=100)
uncovered = _mm_feature_mixed(offset=300, length=100)
@@ -129,8 +127,7 @@ def test_strip_covered_mm_data_shm_address_item() -> None:
"""SHM address descriptors must survive stripping even when covered: the
worker needs the address to resolve the payload and to acknowledge the
sender's writer reference. Stripping it crashed the SHM receiver cache on
the second identical request (vllm-project/vllm#54994).
"""
the second identical request (vllm-project/vllm#54994)."""
address_item = MultiModalKwargsItem(
{
"address": MultiModalFieldElem(data=4096, field=MultiModalBatchedField()),
+25 -50
View File
@@ -126,8 +126,7 @@ def make_kv_cache_manager(kv_cache_config: KVCacheConfig, **kwargs) -> KVCacheMa
"""Build a ``KVCacheManager``, deriving ``scheduler_block_size`` from the
config (LCM of group block sizes) unless explicitly provided. This mirrors
``resolve_kv_cache_block_sizes`` for the non-context-parallel case used by
these tests, so callers don't have to pass it at every site.
"""
these tests, so callers don't have to pass it at every site."""
kwargs.setdefault(
"scheduler_block_size",
lcm(*(g.kv_cache_spec.block_size for g in kv_cache_config.kv_cache_groups)),
@@ -3330,8 +3329,7 @@ def test_kv_cache_events_with_lora(blocks_to_cache: int):
@pytest.mark.parametrize("group_id", [0, 1, 2])
def test_block_stored_event_group_idx(group_id: int):
"""Test BlockStored events emitted by cache_full_blocks carry the correct
group_idx.
"""
group_idx."""
block_size = 4
num_tokens = block_size * 2
@@ -3624,8 +3622,7 @@ def test_block_removed_event_group_idx(group_id: int):
def test_emit_cached_block_events():
"""emit_cached_block_events emits one BlockStored for already-cached
(reused) prefix blocks, carrying the correct group_idx /
parent_block_hash / token_ids, and without mutating block state.
"""
parent_block_hash / token_ids, and without mutating block state."""
block_size = 4
num_cached_blocks = 3
kv_cache_group_id = 1
@@ -3738,8 +3735,7 @@ def test_emit_cached_block_events_zero_cached():
def test_eagle_enabled_removes_last_block():
"""Verify Eagle does NOT remove blocks when request
length is divisible by block size.
"""
length is divisible by block size."""
block_size = 16
manager = make_kv_cache_manager(
make_kv_cache_config(block_size, num_blocks=10),
@@ -4142,8 +4138,7 @@ def test_hybrid_cache_blocks_swa_tail_window_only():
(its right-to-left scan stops once a contiguous match is found). Blocks
earlier in the segment can never serve a hit, so
``HybridKVCacheCoordinator.cache_blocks`` should skip them rather than
polluting the prefix-cache hash map.
"""
polluting the prefix-cache hash map."""
block_size = 8
# Full attn block_size=32, SWA block_size=8, sw=8 -> lcm=32.
# tail = ceil(7/8) = 1; per_segment = 32/8 = 4.
@@ -4214,8 +4209,7 @@ def test_hybrid_cache_blocks_clamped_to_lcm():
Chunks past the last lcm-aligned boundary can never participate in a
cache hit (find_longest_cache_hit always returns lcm-aligned hits), so
caching them only pollutes the prefix-cache hash map and keeps blocks
on the LRU list that could otherwise return to the free pool.
"""
on the LRU list that could otherwise return to the free pool."""
block_size = 16
# Full attn block_size=32, SWA block_size=16 -> lcm=32.
kv_cache_config = KVCacheConfig(
@@ -4350,8 +4344,7 @@ def test_hybrid_local_kv_retention_interval_aligns_in_manager():
)
def test_hybrid_local_kv_retention_interval_rejects_invalid(interval, expected_match):
"""A retention interval that is negative or not a multiple of
scheduler_block_size errors out at construction time.
"""
scheduler_block_size errors out at construction time."""
block_size = 8
kv_cache_config = KVCacheConfig(
num_blocks=100,
@@ -4821,8 +4814,7 @@ def test_block_lookup_cache_multi_blocks_per_key():
def test_can_fit_full_sequence_swa_cap_admits_long_prompt():
"""Hybrid full+SWA model with a pool sized at the startup minimum should
admit a prompt longer than the SWA cap, because SlidingWindowManager
recycles blocks during chunked prefill (issue #39734).
"""
recycles blocks during chunked prefill (issue #39734)."""
block_size = 16
sliding_window = 4 * block_size # 64 tokens
max_num_batched_tokens = 8 * block_size # 128 tokens
@@ -4917,8 +4909,7 @@ def test_full_sequence_admission_reserves_device_blocks_for_immediate_chunk(
def test_can_fit_full_sequence_full_attention_still_gates_oversized():
"""The cap only loosens the SWA group; a prompt that exceeds the
full-attention pool capacity must still be rejected.
"""
full-attention pool capacity must still be rejected."""
block_size = 16
sliding_window = 4 * block_size
max_num_batched_tokens = 8 * block_size
@@ -5037,8 +5028,7 @@ def _take_free_blocks(manager: KVCacheManager, num_blocks: int) -> list[KVCacheB
without removing them. These ref_cnt==0 blocks stand in for evictable
cache-hit blocks left behind by a previous (e.g. preempted) request, and
sitting at the head guarantees a later group's external ``get_new_blocks``
would contend for them on unpatched code (issue #33775).
"""
would contend for them on unpatched code (issue #33775)."""
blocks: list[KVCacheBlock] = []
head = manager.block_pool.free_block_queue.fake_free_list_head
for _ in range(num_blocks):
@@ -5049,8 +5039,7 @@ def _take_free_blocks(manager: KVCacheManager, num_blocks: int) -> list[KVCacheB
def _assert_no_double_allocation(manager: KVCacheManager, req_id: str) -> None:
"""No physical block may be handed out twice across groups, and every
block referenced by the request must have a live ref_cnt.
"""
block referenced by the request must have a live ref_cnt."""
block_ids = manager.get_blocks(req_id).get_block_ids()
flat = [block_id for group in block_ids for block_id in group]
assert len(set(flat)) == len(flat), "Block IDs are not unique across groups"
@@ -5077,8 +5066,7 @@ def _cross_group_cache_hit(
) -> Request:
"""Allocate ``req_id`` with a per-group local prefix hit plus external
(connector) computed tokens, driving the coordinator's two-phase path.
Returns the allocated request so callers can free it (e.g. to preempt).
"""
Returns the allocated request so callers can free it (e.g. to preempt)."""
block_size = _two_phase_block_size(manager)
hit_blocks = _take_free_blocks(manager, num_groups * local_blocks_per_group)
cache_hit = KVCacheBlocks(
@@ -5173,8 +5161,7 @@ def test_swa_free_split_keeps_cached_tail_ahead_of_scratch():
"""Dense retention: freeing an SWA request must place its
uncached scratch blocks at the front of the free queue (recycled first)
and keep its cached checkpoint blocks at the back (retained for prefix
hits). This split is always-on, independent of the retention interval.
"""
hits). This split is always-on, independent of the retention interval."""
block_size = 8
kv_cache_config = KVCacheConfig(
num_blocks=100,
@@ -5285,8 +5272,7 @@ def _make_pure_swa_manager(block_size, sliding_window, num_blocks=100, **kwargs)
def test_pure_swa_retention_interval_caches_sparse_tails():
"""Sparse retention must work for a pure-SWA single-group model, not just
hybrid models: only the per-interval tails plus the latest replay tail are
cached, and a replay still hits the latest replayable boundary.
"""
cached, and a replay still hits the latest replayable boundary."""
block_size = 16
manager = _make_pure_swa_manager(
block_size, sliding_window=block_size, retention_interval=64
@@ -5358,8 +5344,7 @@ def test_pure_swa_retention_latest_only():
def test_pure_swa_dense_retention_caches_all():
"""With retention set to ``None``, a pure-SWA model keeps dense behavior:
every block boundary is a potential hit, so all blocks are cached.
"""
every block boundary is a potential hit, so all blocks are cached."""
block_size = 16
manager = _make_pure_swa_manager(block_size, sliding_window=block_size)
@@ -5388,8 +5373,7 @@ def test_mamba_reachable_block_mask_sparsifies_retention():
"""Mamba state-snapshot retention: with a configured retention interval,
the manager keeps one cached state per interval-sized segment (plus the
latest replay boundary) instead of a snapshot per block, which is what
lets a small attention block_size avoid Mamba dominating the KV pool.
"""
lets a small attention block_size avoid Mamba dominating the KV pool."""
from vllm.v1.core.single_type_kv_cache_manager import MambaManager
block_size = 16
@@ -5427,8 +5411,7 @@ def test_mamba_reachable_block_mask_pins_shared_prefix():
"""A Marconi-detected shared prefix (``shared_prefix_boundary``) lands before
``num_prompt`` so the replay-boundary rule alone would drop it. The mask must
pin the single state block ending on that boundary so sparse retention does
not defeat cross-request shared-prefix reuse.
"""
not defeat cross-request shared-prefix reuse."""
from vllm.v1.core.single_type_kv_cache_manager import MambaManager
block_size = 16
@@ -5476,8 +5459,7 @@ def test_mamba_shared_prefix_survives_zero_retention():
detection) keeps its Mamba state block cached under
``prefix_cache_retention_interval=0``, which otherwise retains only the
end-of-prompt replay boundary. Without this, a shared prefix (junction
before ``num_prompt``) would be recomputed by every sharing request.
"""
before ``num_prompt``) would be recomputed by every sharing request."""
block_size = 16
# 16-block (256-token) prompt; replay boundary is block 240 // 16 - 1 = 14.
@@ -5519,8 +5501,7 @@ def test_mamba_shared_prefix_reuse_under_zero_retention():
detecting request must stay reusable by a later request under
``prefix_cache_retention_interval=0``. Without the pin the junction is
masked out and the later request misses; with it (and under dense) the reuse
is preserved.
"""
is preserved."""
block_size = 16
def last_req_hit(retention, pin):
@@ -5603,8 +5584,7 @@ def test_swa_reachable_block_mask_final_partial_segment():
def test_swa_reachable_block_mask_pins_shared_prefix():
"""SWA analog of the Mamba pin: the shared-prefix junction must keep the
``need``-block sliding-window tail ending on that boundary (not a single
block), so a windowed hit can land there under sparse retention.
"""
block), so a windowed hit can land there under sparse retention."""
from vllm.v1.core.single_type_kv_cache_manager import SlidingWindowManager
block_size = 16
@@ -5649,8 +5629,7 @@ def test_swa_reachable_block_mask_pins_shared_prefix():
def test_swa_reachable_block_mask_with_dcp_scaling():
"""DCP shards each block's KV across ranks, scaling the effective block size.
Verify that dcp_world_size > 1 scales block size in reachability calculations,
producing different masks than dcp_world_size=1.
"""
producing different masks than dcp_world_size=1."""
from vllm.v1.core.single_type_kv_cache_manager import SlidingWindowManager
block_size = 16
@@ -5709,8 +5688,7 @@ def test_swa_reachable_block_mask_sub_block_alignment_is_dense(
multiple of the DCP-scaled block size) cannot be represented exactly. This
happens for hybrid offloading (e.g. Gemma), where alignment_tokens is the
full-attention chunk size and need not divide the SWA block size. The mask
must fall back to dense (None) rather than raise.
"""
must fall back to dense (None) rather than raise."""
from vllm.v1.core.single_type_kv_cache_manager import SlidingWindowManager
spec = SlidingWindowSpec(
@@ -5738,8 +5716,7 @@ def test_mamba_reachable_block_mask_ignores_dcp():
"""Mamba uses TP, not DCP: each rank holds the full recurrent state, so a
state block spans kv_cache_spec.block_size tokens regardless of DCP. The
mask must not scale by dcp_world_size, so retention granularity is
identical for any DCP world size.
"""
identical for any DCP world size."""
from vllm.v1.core.single_type_kv_cache_manager import MambaManager
spec = MambaSpec(
@@ -5773,8 +5750,7 @@ def test_mamba_reachable_block_mask_ignores_dcp():
def test_mamba_reachable_block_mask_large_dcp_stays_sparse():
"""A large DCP world size must not scale the Mamba block size, so it can
never collapse sparse retention into dense caching.
"""
never collapse sparse retention into dense caching."""
from vllm.v1.core.single_type_kv_cache_manager import MambaManager
spec = MambaSpec(
@@ -5809,8 +5785,7 @@ def test_swa_shared_prefix_reuse_under_zero_retention():
"""SWA cross-request analog: a partial shared prefix's sliding-window tail
must stay reusable under ``prefix_cache_retention_interval=0``. Without
the pin the junction window is masked out and a later request misses; with
it (and under dense) reuse is preserved.
"""
it (and under dense) reuse is preserved."""
block_size = 16
def last_req_hit(retention, pin):
+28 -56
View File
@@ -538,8 +538,7 @@ def test_throttle_defers_inflight_prefill_chunk():
"""DP prefill balancing throttles ALL prefill compute on a throttled step,
not just new admissions: an in-progress (chunked) prefill already in the
running queue is also deferred, so the step runs decode-only, while a
separate decode keeps being scheduled.
"""
separate decode keeps being scheduled."""
scheduler = create_scheduler(
max_num_seqs=16, max_num_batched_tokens=50, enable_chunked_prefill=True
)
@@ -593,8 +592,7 @@ def test_throttle_capacity_bound_guard_admits():
"""Saturation guard: if a cadence-aligned release step cannot drain the
waiting prefill queue (it ran out of token budget), the throttle backs off on
the next step so the backlog cannot grow into a TTFT avalanche -- prefills are
admitted even though throttle_prefills is set.
"""
admitted even though throttle_prefills is set."""
scheduler = create_scheduler(
max_num_seqs=16, max_num_batched_tokens=200, enable_chunked_prefill=True
)
@@ -1293,8 +1291,7 @@ def test_pending_hisparse_reclamation_defers_preemption(monkeypatch):
def test_prefix_cache_query_not_inflated_by_connector_defer():
"""The GPU prefix-cache query is recorded at admission, so a request the
connector defers several times is counted once, not once per retry.
"""
connector defers several times is counted once, not once per retry."""
num_defers_before_matching = 3
scheduler = create_scheduler(
enable_prefix_caching=True,
@@ -1322,8 +1319,7 @@ def test_prefix_cache_query_not_inflated_by_connector_defer():
def test_preemption_re_records_prefix_cache_query():
"""A preempted request re-enters the lookup on resume, so its recomputation
is counted again into the preempted stats.
"""
is counted again into the preempted stats."""
scheduler = create_scheduler(enable_prefix_caching=True)
request = create_requests(num_requests=1)[0]
scheduler.add_request(request)
@@ -1359,8 +1355,7 @@ def test_preemption_re_records_prefix_cache_query():
def test_prefix_cache_stats_not_recorded_when_caching_disabled():
"""With prefix caching off there is no local lookup, so admitting a request
records no phantom miss.
"""
records no phantom miss."""
scheduler = create_scheduler(enable_prefix_caching=False)
for request in create_requests(num_requests=2):
scheduler.add_request(request)
@@ -1374,8 +1369,7 @@ def test_prefix_cache_stats_not_recorded_when_caching_disabled():
def test_prefix_cache_stats_counted_once_for_retried_then_scheduled_request():
"""A real cache hit rejected once by allocate_slots and admitted on the next
step is counted exactly once, hits included.
"""
step is counted exactly once, hits included."""
block_size = 16
scheduler = create_scheduler(
enable_prefix_caching=True,
@@ -3094,8 +3088,7 @@ def create_requests_with_priority(
def test_priority_scheduling_basic_ordering():
"""Test that requests are scheduled in priority order
(lower value = higher priority).
"""
(lower value = higher priority)."""
scheduler = create_scheduler_with_priority()
# Create requests with different priorities
@@ -3124,8 +3117,7 @@ def test_priority_scheduling_basic_ordering():
def test_priority_scheduling_arrival_time_tiebreaker():
"""Test that arrival time is used
as tiebreaker when priorities are equal.
"""
as tiebreaker when priorities are equal."""
scheduler = create_scheduler_with_priority()
# Create requests with same priority but different arrival times
@@ -3282,8 +3274,7 @@ def test_priority_scheduling_preemption():
def test_priority_scheduling_no_preemption_when_space_available():
"""Test that preemption doesn't happen
when there's space for new requests.
"""
when there's space for new requests."""
scheduler = create_scheduler_with_priority(
max_num_seqs=3, # Allow 3 concurrent requests
max_num_batched_tokens=200, # Sufficient token budget
@@ -3336,8 +3327,7 @@ def test_priority_scheduling_no_preemption_when_space_available():
def test_priority_scheduling_preemption_victim_selection():
"""Test that the correct victim is selected for
preemption based on priority and arrival time.
"""
preemption based on priority and arrival time."""
# This test verifies the priority-based victim selection logic
# by checking the waiting queue order after adding requests with different
# priorities
@@ -3457,8 +3447,7 @@ def test_priority_scheduling_waiting_queue_order():
def test_priority_scheduling_fcfs_fallback():
"""Test that FCFS behavior is maintained when all
requests have same priority.
"""
requests have same priority."""
scheduler = create_scheduler_with_priority()
# Create requests with same priority but different arrival times
@@ -3527,8 +3516,7 @@ def test_priority_scheduling_with_limited_slots():
def test_priority_scheduling_heap_property():
"""Test that the waiting queue maintains heap
property for priority scheduling.
"""
property for priority scheduling."""
scheduler = create_scheduler_with_priority(
max_num_seqs=1, # Only one request can run at a time
)
@@ -3753,8 +3741,7 @@ def test_priority_scheduling_preemption_and_resumption_when_out_of_kv(
use_ec_connector, ec_role, use_v2_model_runner
):
"""Test that priority scheduling preempts lower priority requests
when out of KV cache space.
"""
when out of KV cache space."""
# Create scheduler with very limited memory to force preemption
scheduler = create_scheduler_with_priority(
max_num_seqs=2, # Allow multiple requests
@@ -3899,8 +3886,7 @@ def test_chunked_prefill_disabled_for_encoder_decoder(
enable_chunked_prefill: bool, is_encoder_decoder: bool, expect_enabled: bool
) -> None:
"""Validate that chunked prefill is appropriately disabled for
encoder-decoder models.
"""
encoder-decoder models."""
scheduler_config = SchedulerConfig(
enable_chunked_prefill=enable_chunked_prefill,
is_encoder_decoder=is_encoder_decoder,
@@ -3929,8 +3915,7 @@ def _validate_chunked_prefill_settings_for_encoder_decoder(
scheduler_config: SchedulerConfig, is_encoder_decoder: bool, expect_enabled: bool
) -> None:
"""Validate chunked prefill settings in the scheduler config for
encoder-decoder models.
"""
encoder-decoder models."""
assert scheduler_config.enable_chunked_prefill is expect_enabled
if is_encoder_decoder:
# Encoder-decoder models should automatically disable chunked multimodal
@@ -4020,8 +4005,7 @@ def _assert_right_encoder_inputs(
):
"""Verify that requests/mm_hashes should (not) in scheduled encoder input
If check_exist is False, this function returns True
if requests are NOT in encoder inputs
"""
if requests are NOT in encoder inputs"""
# Get the scheduled encoder inputs
# NOTE: scheduled_encoder_inputs is a dictionary with request id as key
scheduled_encoder_inputs = output.scheduled_encoder_inputs
@@ -4182,8 +4166,7 @@ def test_ec_connector_text_only_request(use_kv_connector):
@pytest.mark.parametrize("use_kv_connector", [False, True])
def test_ec_connector_cache_hit_external_load(use_kv_connector):
"""Test ec_consumer loads from external cache when hit.
A normal basic operation for EPD disaggrgation
"""
A normal basic operation for EPD disaggrgation"""
scheduler = create_scheduler(
model="llava-hf/llava-1.5-7b-hf",
enable_prefix_caching=True,
@@ -4630,8 +4613,7 @@ def test_priority_scheduling_ec_connector_preemption_and_resumption(
cache_exist, use_kv_connector
):
"""Test that priority scheduling preempts lower priority requests
when out of KV cache space.
"""
when out of KV cache space."""
# Create scheduler with very limited memory to force preemption
scheduler = create_scheduler_with_priority(
model="llava-hf/llava-1.5-7b-hf",
@@ -5300,8 +5282,7 @@ def test_scheduler_kv_connector_stats():
def test_ec_connector_update_connector_output_called():
"""Test that worker-side EC connector output is forwarded to the
EC connector's update_connector_output hook.
"""
EC connector's update_connector_output hook."""
scheduler = create_scheduler(
model="llava-hf/llava-1.5-7b-hf",
use_ec_connector=True,
@@ -5506,8 +5487,7 @@ def test_variable_length_cross_attn_block_allocation():
def test_cross_attn_blocks_not_over_allocated():
"""Test that cross-attention blocks are not over-allocated compared to
what each request actually needs.
"""
what each request actually needs."""
from math import ceil
block_size = 16
@@ -5603,8 +5583,7 @@ def test_cross_attn_blocks_not_under_allocated():
def test_cross_attn_zero_blocks_without_encoder_inputs():
"""Test that requests without encoder inputs get zero cross-attention
blocks, even when the scheduler is configured for encoder-decoder.
"""
blocks, even when the scheduler is configured for encoder-decoder."""
block_size = 16
scheduler = _create_encoder_decoder_scheduler(block_size=block_size)
@@ -5709,8 +5688,7 @@ def test_free_encoder_inputs_respects_unconfirmed_placeholders():
evicted entry and crashes the engine with "Encoder cache miss". The
scheduler must retain the input until the *confirmed* progress
(num_computed_tokens - num_output_placeholders) passes the range end, so
that no pending rejection can rewind into the range.
"""
that no pending rejection can rewind into the range."""
scheduler = create_scheduler(
model="llava-hf/llava-1.5-7b-hf",
num_speculative_tokens=3,
@@ -5821,8 +5799,7 @@ def test_free_encoder_inputs_defers_for_eagle_lookahead():
position so the drafter's +1 look-ahead mm-embedding gather (which reads one
position past the target's computed range) still finds it cached. This is
the primary mechanism that prevents the drafter "Encoder cache miss"; the
worker-side token-embedding fallback is only a backstop.
"""
worker-side token-embedding fallback is only a backstop."""
scheduler = create_scheduler(model="llava-hf/llava-1.5-7b-hf")
# create_scheduler only builds ngram spec configs; force the eagle path that
# _free_encoder_inputs keys off (its read-ahead deferral).
@@ -5852,8 +5829,7 @@ def test_free_encoder_inputs_defers_for_eagle_lookahead():
def test_free_encoder_inputs_unchanged_without_spec_decode():
"""Without speculative decoding, encoder inputs are freed as soon as
num_computed_tokens passes the placeholder range, as before.
"""
num_computed_tokens passes the placeholder range, as before."""
scheduler = create_scheduler(model="llava-hf/llava-1.5-7b-hf")
mm_positions = [[PlaceholderRange(offset=50, length=100)]]
request = create_requests(
@@ -5884,8 +5860,7 @@ def test_encoder_cache_retained_across_preemption_and_resume():
encoder input must pull the still-cached entry back out of `freeable`
without scheduling a recompute, keeping the scheduler and worker
consistent. The spec-rollback retention margin does not gate this path,
so it is covered separately here.
"""
so it is covered separately here."""
scheduler = create_scheduler(
model="llava-hf/llava-1.5-7b-hf",
num_speculative_tokens=3,
@@ -5929,8 +5904,7 @@ def test_encoder_cache_recomputed_when_evicted_during_preemption():
pressure before it resumes, the scheduler reports the mm_hash as freed
(so the worker drops it) and a resume must schedule a recompute rather
than assume the worker still holds it. check_and_update_cache must
return False so the encoder input is re-scheduled.
"""
return False so the encoder input is re-scheduled."""
scheduler = create_scheduler(
model="llava-hf/llava-1.5-7b-hf",
num_speculative_tokens=3,
@@ -6581,8 +6555,7 @@ def _decode_ready_request(scheduler):
def test_update_draft_token_ids_strips_ngram_padding():
"""ngram_gpu pads unfilled draft slots with -1; manager validation must
strip them before grammar.validate_tokens (which otherwise raises).
"""
strip them before grammar.validate_tokens (which otherwise raises)."""
scheduler = create_scheduler(num_speculative_tokens=4)
request = _decode_ready_request(scheduler)
@@ -6601,8 +6574,7 @@ def test_update_draft_token_ids_strips_ngram_padding():
def test_update_draft_token_ids_in_output_strips_padding():
"""Same guard on the output path; the -1 pad-back for the rejected count
is preserved (only the input to manager validation is stripped).
"""
is preserved (only the input to manager validation is stripped)."""
scheduler = create_scheduler(num_speculative_tokens=4)
request = _decode_ready_request(scheduler)
@@ -97,8 +97,7 @@ def test_num_in_flight_tokens_accounting():
def test_swa_free_waits_for_in_flight_step():
"""Async: out-of-window blocks stay allocated until the step that still
reads them has been processed.
"""
reads them has been processed."""
scheduler = _create_swa_scheduler(async_scheduling=True)
request = create_requests(
num_requests=1, num_tokens=NUM_PROMPT_TOKENS, block_size=BLOCK_SIZE
@@ -133,8 +132,7 @@ def test_swa_free_waits_for_in_flight_step():
def test_swa_free_immediate_when_sync():
"""Sync: no in-flight step at schedule time, frees happen at the first
decode allocation as before.
"""
decode allocation as before."""
scheduler = _create_swa_scheduler(async_scheduling=False)
request = create_requests(
num_requests=1, num_tokens=NUM_PROMPT_TOKENS, block_size=BLOCK_SIZE
@@ -173,8 +171,7 @@ def test_swa_admission_cap_accounts_for_overlapping_batches():
def test_chunked_local_free_waits_for_in_flight_step():
"""Chunked-local attention frees whole chunks left of the current one, and
is exposed to the same load-WAR: with async scheduling those chunks must
stay allocated until the in-flight step that still reads them settles.
"""
stay allocated until the in-flight step that still reads them settles."""
scheduler = _create_chunked_scheduler(async_scheduling=True)
request = create_requests(
num_requests=1, num_tokens=NUM_PROMPT_TOKENS, block_size=BLOCK_SIZE
@@ -229,8 +226,7 @@ def test_connector_finish_frees_on_settled_basis():
is handed to a KV connector (simple CPU offload / NIXL store), must use the
same processed-token basis. Otherwise the connector reads/hands off a block
the still-in-flight step is optimistically counted as done with, which is
the load-path WAR this fix closes.
"""
the load-path WAR this fix closes."""
scheduler = create_scheduler(
block_size=BLOCK_SIZE,
async_scheduling=True,
@@ -80,8 +80,7 @@ def test_piecewise_capture_builds_fresh_metadata_for_both_passes():
@pytest.fixture(autouse=True)
def _reset_breakable_tls():
"""Defensively clear thread-local capture state between tests so a
failure in one test can't leak "nested capture" errors into the next.
"""
failure in one test can't leak "nested capture" errors into the next."""
from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphCapture
BreakableCUDAGraphCapture._tls.active = None
@@ -291,8 +290,7 @@ def test_capture_replay_matches_eager_simple(cuda_capture_stream):
def test_decorator_breaks_when_invoked_inside_capture(cuda_capture_stream):
"""Verify @eager_break_during_capture correctly routes through
add_eager when inside a capture context, and runs straight through
when there's no active capture.
"""
when there's no active capture."""
from vllm.compilation.breakable_cudagraph import (
BreakableCUDAGraphCapture,
eager_break_during_capture,
@@ -554,8 +552,7 @@ def _mock_vllm_config():
def test_wrapper_falls_through_on_runtime_mode_mismatch():
"""A PIECEWISE-scoped wrapper runs other modes eagerly, capturing
nothing, so an outer CUDAGraphWrapper can own them.
"""
nothing, so an outer CUDAGraphWrapper can own them."""
from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphWrapper
from vllm.config import CUDAGraphMode
from vllm.forward_context import BatchDescriptor, set_forward_context
@@ -275,8 +275,7 @@ def test_rms_norm_per_block_quant_batch_invariant(
hidden_size: int, dtype: torch.dtype, seed: int
):
"""C++ ``rms_norm_per_block_quant`` must be batch invariant across the
block threshold (compiled fp8 block-quant path; block pinned to 512).
"""
block threshold (compiled fp8 block-quant path; block pinned to 512)."""
import vllm._custom_ops as ops
device = torch.device(DEVICE_TYPE)
+4 -8
View File
@@ -547,8 +547,7 @@ async def test_dp_pause_keep_then_resume(expert_parallel: bool):
async def test_dp_pause_keep_race_staggered_engines():
"""Race: send pause(keep) to engine 0, then add two requests,
then pause(keep) to engine 1. Ensures no deadlock when pause
requests are staggered and requests arrive in between.
"""
requests are staggered and requests arrive in between."""
if DP_SIZE != 2:
pytest.skip("test_dp_pause_keep_race_staggered_engines requires DP_SIZE=2")
@@ -718,8 +717,7 @@ async def test_dp_pause_barrier_request_deadlock():
async def test_dp_pause_wait_mode_drains_in_flight():
"""mode="wait" through the DP consensus path: the pause lets the
in-flight request run to completion while the consensus is pending,
resolves only once drained, and the engine generates again on resume.
"""
resolves only once drained, and the engine generates again on resume."""
with ExitStack() as after:
engine = AsyncLLM.from_engine_args(_get_dp_pause_engine_args(True))
after.callback(engine.shutdown)
@@ -750,8 +748,7 @@ async def test_dp_pause_wait_mode_drains_in_flight():
async def test_dp_pause_while_asleep():
"""Pausing a sleeping DP engine: sleeping ranks skip dummy batches yet
must still reach the pause consensus, and the completion barrier must
be harmless on workers whose memory is unmapped.
"""
be harmless on workers whose memory is unmapped."""
with ExitStack() as after:
engine = AsyncLLM.from_engine_args(_get_dp_pause_engine_args(True))
after.callback(engine.shutdown)
@@ -787,8 +784,7 @@ async def test_dp_pause_completion_implies_device_idle(
"""A resolved pause future promises an idle device. Inflate the dummy
batches the pause consensus manufactures so any work the pause fails to
wait on stays visible, then require a quiet worker stream the moment
pause_generation() returns.
"""
pause_generation() returns."""
with ExitStack() as after:
# The probes below ship functions through collective_rpc.
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
+1 -2
View File
@@ -24,8 +24,7 @@ TP_SIZE = int(os.getenv("TP_SIZE", "1"))
class ExternalLBServerManager:
"""Manages data parallel vLLM server instances for external
load balancer testing.
"""
load balancer testing."""
def __init__(
self,
+1 -2
View File
@@ -30,8 +30,7 @@ DP_SIZE_LOCAL = DP_SIZE // NUM_NODES # 2 ranks per node
class HybridLBServerManager:
"""Manages hybrid data parallel vLLM server instances where each node
runs a single logical API server that balances requests only to the
DP engines running on that same node.
"""
DP engines running on that same node."""
def __init__(
self,
+3 -6
View File
@@ -107,8 +107,7 @@ async def _run_request_bursts(
class MultinodeInternalLBServerManager:
"""Manages multi-node data parallel vLLM server instances for internal
load balancer testing using --headless mode.
"""
load balancer testing using --headless mode."""
def __init__(
self,
@@ -242,8 +241,7 @@ class MultinodeInternalLBServerManager:
class APIOnlyServerManager:
"""Manages API-only server (Node 0) and headless engines server (Node 1)
for testing separated API server and engine configuration.
"""
for testing separated API server and engine configuration."""
def __init__(
self,
@@ -645,8 +643,7 @@ async def test_api_only_multinode_dp_completion_streaming(
model_name: str,
) -> None:
"""Test API-only server streaming with all engines on separate
headless server.
"""
headless server."""
prompt = "What is an LLM?"
async def make_streaming_request():
+4 -8
View File
@@ -82,8 +82,7 @@ async def _generate(engine: AsyncLLM, prompt: str, max_tokens: int) -> int:
@pytest.mark.parametrize("async_scheduling", [True, False])
async def test_pp_dp_v2_low_concurrency(async_scheduling: bool):
"""A single in-flight request at a time, repeated, to exercise the
PP slot ring under empty batches between decodes.
"""
PP slot ring under empty batches between decodes."""
with ExitStack() as after:
engine = AsyncLLM.from_engine_args(_engine_args(async_scheduling))
after.callback(engine.shutdown)
@@ -97,8 +96,7 @@ async def test_pp_dp_v2_low_concurrency(async_scheduling: bool):
@pytest.mark.parametrize("async_scheduling", [True, False])
async def test_pp_dp_v2_mid_concurrency(async_scheduling: bool):
"""64 concurrent requests, staggered, to exercise the steady-state
DP all-reduce + PP slot-ring path.
"""
DP all-reduce + PP slot-ring path."""
with ExitStack() as after:
engine = AsyncLLM.from_engine_args(_engine_args(async_scheduling))
after.callback(engine.shutdown)
@@ -114,8 +112,7 @@ async def test_pp_dp_v2_mid_concurrency(async_scheduling: bool):
@pytest.mark.asyncio
async def test_pp_dp_v2_abort_mid_decode():
"""Cancel half the in-flight requests mid-stream and confirm the
engine survives the abort storm.
"""
engine survives the abort storm."""
with ExitStack() as after:
engine = AsyncLLM.from_engine_args(_engine_args(async_scheduling=True))
after.callback(engine.shutdown)
@@ -155,8 +152,7 @@ async def test_pp_dp_v2_abort_mid_decode():
@pytest.mark.asyncio
async def test_pp_dp_v2_pause_resume():
"""Pause an engine with a request in flight, then resume and confirm
new requests still work.
"""
new requests still work."""
with ExitStack() as after:
engine = AsyncLLM.from_engine_args(_engine_args(async_scheduling=True))
after.callback(engine.shutdown)
@@ -48,8 +48,7 @@ def test_without_spec_decoding(
monkeypatch: pytest.MonkeyPatch,
):
"""Test consistency of combos of async scheduling, preemption,
uni/multiproc executor, prefill chunking.
"""
uni/multiproc executor, prefill chunking."""
struct_outputs = StructuredOutputsParams(json=sample_json_schema)
test_sampling_params: list[dict[str, Any]] = [
dict(),
@@ -205,8 +204,7 @@ def run_tests(
test_sampling_params: list[dict[str, Any]],
):
"""Test consistency of combos of async scheduling, preemption,
uni/multiproc executor with spec decoding.
"""
uni/multiproc executor with spec decoding."""
# Flex attention supports float32.
attention_config = {"backend": "FLEX_ATTENTION"}
@@ -72,8 +72,7 @@ def test_backend_per_kind_splits_groups(backend_per_kind, monkeypatch):
)
def test_backend_per_kind_overrides_global_backend(monkeypatch):
"""A per-kind entry wins over the global ``backend`` for its kind; other
kinds fall back to the global backend.
"""
kinds fall back to the global backend."""
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
llm = LLM(
model=MODEL,
+2 -4
View File
@@ -34,8 +34,7 @@ def test_decoder_max_context_length_validation(
max_tokens: int,
) -> None:
"""Check vLLM decoder model input validation for edge cases where
the prompt length is (almost) equal to the max model length.
"""
the prompt length is (almost) equal to the max model length."""
prompt_ids = [[43] * prompt_len]
with vllm_runner(
@@ -72,8 +71,7 @@ def test_auto_fit_max_model_len_rejects_oversized_input(
"""When max_model_len='auto' and KV cache memory is very limited,
the engine auto-fits max_model_len to a small value. The frontend
must see this reduced value and reject prompts that exceed it,
rather than accepting them and hanging.
"""
rather than accepting them and hanging."""
# Use a small KV cache budget to force auto-fit to a small
# max_model_len. Pin block_size=16 so the budget is independent
# of the platform's default block size. One block for this model is
+1 -2
View File
@@ -172,8 +172,7 @@ def llm_v1():
def test_min_tokens_with_terminal_structured_output(llm_v1: LLM):
"""A grammar whose only legal continuation is EOS must stop under
min_tokens instead of sampling outside the grammar.
"""
min_tokens instead of sampling outside the grammar."""
params = SamplingParams(
temperature=GREEDY,
min_tokens=20,
@@ -7,8 +7,7 @@ slightly different kernel/collective states (measured: same-state boots are
bitwise identical across the two modes; cross-state boots differ by up to
~0.25 logprob and can flip a near-tie token). A real sharding bug — rows
routed to the wrong request, misaligned gathers — produces many-nats
divergence and scrambled top-k sets, far outside these bounds.
"""
divergence and scrambled top-k sets, far outside these bounds."""
import pytest
@@ -107,8 +106,7 @@ def _assert_logprob_dicts_close(ref_lps, out_lps, what: str) -> None:
@multi_gpu_test(num_gpus=2)
def test_sharded_sampling_outputs_match(monkeypatch: pytest.MonkeyPatch):
"""Generation, logprobs, and prompt logprobs match between batch-sharded
sampling and the replicated fallback, up to measured boot-state noise.
"""
sampling and the replicated fallback, up to measured boot-state noise."""
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
# Required for the collective_rpc mode check below.
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
@@ -11,8 +11,7 @@ from ...utils import compute_acceptance_len, get_test_prompts
@single_gpu_only
def test_synthetic_acceptance_rate(vllm_runner):
"""Verify that synthetic rejection sampling produces an acceptance
length close to the requested mean acceptance length.
"""
length close to the requested mean acceptance length."""
num_spec_tokens = 3
expected_acceptance_len = 1.875
tolerance = 0.15
@@ -167,8 +167,7 @@ def test_draft_model_tensor_parallelism(vllm_runner):
@multi_gpu_only(num_gpus=2)
def test_draft_model_engine_args_tensor_parallelism():
"""Ensure the vllm_config for the draft model is created correctly,
and independently of the target model (quantization, TP, etc.)
"""
and independently of the target model (quantization, TP, etc.)"""
engine_args = EngineArgs(
model="Qwen/Qwen3-1.7B-FP8", # <<< tgt quantized
tensor_parallel_size=2,
@@ -199,8 +198,7 @@ def test_draft_model_engine_args_tensor_parallelism():
def _apply_draft_moe_backend(vllm_config: VllmConfig) -> VllmConfig:
"""Replicate SpecDecodeBaseProposer._create_draft_vllm_config logic
so we can test it without instantiating a full proposer.
"""
so we can test it without instantiating a full proposer."""
spec_cfg = vllm_config.speculative_config
if spec_cfg.moe_backend is not None:
return replace(
@@ -225,8 +223,7 @@ def _platform_moe_backend(monkeypatch: pytest.MonkeyPatch) -> MoEBackend:
def test_draft_model_moe_backend_override(monkeypatch: pytest.MonkeyPatch):
"""When moe_backend is set in speculative_config, the draft VllmConfig
should use it while the target keeps its own setting.
"""
should use it while the target keeps its own setting."""
target_moe_backend = _platform_moe_backend(monkeypatch)
engine_args = EngineArgs(
model="Qwen/Qwen3-1.7B",
@@ -251,8 +248,7 @@ def test_draft_model_moe_backend_override(monkeypatch: pytest.MonkeyPatch):
def test_draft_model_moe_backend_inherits_target(monkeypatch: pytest.MonkeyPatch):
"""When moe_backend is not set in speculative_config, the draft should
inherit the target's moe_backend.
"""
inherit the target's moe_backend."""
target_moe_backend = _platform_moe_backend(monkeypatch)
engine_args = EngineArgs(
model="Qwen/Qwen3-1.7B",
@@ -275,8 +271,7 @@ def test_draft_model_moe_backend_inherits_target(monkeypatch: pytest.MonkeyPatch
def test_draft_model_moe_backend_default_auto():
"""When neither target nor draft set moe_backend explicitly, both should
default to 'auto'.
"""
default to 'auto'."""
engine_args = EngineArgs(
model="Qwen/Qwen3-1.7B",
tensor_parallel_size=1,
@@ -297,8 +292,7 @@ def test_draft_model_moe_backend_default_auto():
def test_draft_model_engine_args_rejects_invalid_tp_argname():
"""The user should pass "draft_tensor_parallel_size" rather than
"tensor_parallel_size". We enforce this with validation.
"""
"tensor_parallel_size". We enforce this with validation."""
engine_args = EngineArgs(
model="Qwen/Qwen3-1.7B",
tensor_parallel_size=1,
@@ -315,8 +309,7 @@ def test_draft_model_engine_args_rejects_invalid_tp_argname():
def assert_draft_model_correctness(args: ArgsTest, vllm_runner):
"""Compare the outputs using and not using speculative decoding.
In the greedy decoding case, the outputs must match EXACTLY.
"""
In the greedy decoding case, the outputs must match EXACTLY."""
test_prompts: list[Messages] = get_messages(
dataset=args.dataset, n=args.num_prompts
)
@@ -28,8 +28,7 @@ def test_mtp_sharded_sampling_equivalence(monkeypatch: pytest.MonkeyPatch):
sampling under MTP spec decoding: the collectives move the same logits
bytes, Gumbel keys derive from (request slot, position, seed), and slot
assignment is rank-deterministic. Both runs here are spec decode with
identical math, so outputs must match exactly.
"""
identical math, so outputs must match exactly."""
tp_size = 2
_skip_if_insufficient_gpus_for_tp(tp_size)
model_name = "Qwen/Qwen3.5-0.8B-Base"
+1 -2
View File
@@ -67,8 +67,7 @@ def test_mtp_speculative_mixed_batch_short_prefill(
vllm_runner, model_name, enable_prefix_caching
):
"""Test to ensure MTP speculative decoding correctly handles
short prefill chunks that fall below the reorder_batch_threshold.
"""
short prefill chunks that fall below the reorder_batch_threshold."""
# Set so large that both prefills will be classified as decodes in a mixed batch
# note, with prefix caching we require chunk_size >= mamba_block_size
chunk_size = 256 if not enable_prefix_caching else 2048
@@ -381,8 +381,7 @@ def test_pin_if_ready_none_when_absent():
def test_pin_if_ready_returns_unpinned_entry_when_not_ready():
"""The producer needs "present but not ready" told apart from "absent":
the first is NACKed as retryable, the second as a miss.
"""
the first is NACKed as retryable, the second as a miss."""
cache = EmbeddingCache(num_blocks=4)
cache.alloc("h", 1) # not-ready
entry = cache.pin_if_ready("h")
@@ -131,8 +131,7 @@ def test_has_cache_item_false_when_not_consumer(monkeypatch):
def test_connector_keys_on_identifier_not_mm_hash(monkeypatch):
"""The connector must key the encoder cache on feature.identifier (what
has_cache_item is called with), NOT feature.mm_hash.
"""
has_cache_item is called with), NOT feature.mm_hash."""
s = _make_scheduler(monkeypatch)
req = _Request([_Feature("PROC_KEY", length=1, identifier="ENC_KEY")])
s.update_state_after_alloc(req, 0)
@@ -202,8 +201,7 @@ def test_loads_only_serves_hashes_touched_this_step(monkeypatch):
def test_repeated_reload_same_step_loads_once(monkeypatch):
"""The same mm_hash requested twice in one step must appear in
meta.loads exactly once.
"""
meta.loads exactly once."""
s = _make_scheduler(monkeypatch)
_seed_cached(s, "a", n_blocks=2)
@@ -226,8 +224,7 @@ def test_load_not_emitted_for_uncached_entry(monkeypatch):
def test_load_pin_protects_blocks_until_completion(monkeypatch):
"""Loaded blocks stay pinned until the worker reports the load memcpy
complete. Observable: a save that needs eviction cannot reclaim a
still-pinned loaded entry, but can once the completion report arrives.
"""
still-pinned loaded entry, but can once the completion report arrives."""
# 2 blocks, both occupied by "a" (ready).
s = _make_scheduler(monkeypatch, num_blocks=2)
_seed_cached(s, "a", n_blocks=2)
@@ -254,8 +251,7 @@ def test_load_pin_protects_blocks_until_completion(monkeypatch):
def _can_evict(s: ECCPUScheduler, mm_hash: str, n_blocks: int) -> bool:
"""Whether a new save of `mm_hash` can claim `n_blocks`, i.e. whether the
blocks currently held by other entries are reclaimable.
"""
blocks currently held by other entries are reclaimable."""
s.update_state_after_alloc(_Request([_Feature(mm_hash, length=n_blocks)]), 0)
meta = s.build_connector_meta(scheduler_output=None)
return mm_hash in meta.saves
@@ -315,8 +311,7 @@ def test_late_report_does_not_release_a_later_load_of_same_hash(monkeypatch):
def test_region_full_skips_save_and_never_blocks(monkeypatch):
"""When the region is fully occupied by pinned entries, new saves are
silently skipped and ensure_cache_available never blocks.
"""
silently skipped and ensure_cache_available never blocks."""
s = _make_scheduler(monkeypatch, num_blocks=1)
_seed_cached(s, "pinned", n_blocks=1)
s._cache.pin("pinned")
@@ -335,8 +330,7 @@ def test_region_full_skips_save_and_never_blocks(monkeypatch):
def test_producer_only_never_emits_loads(monkeypatch):
"""A producer-only scheduler must never populate meta.loads, even when
entries are ready.
"""
entries are ready."""
s = _make_scheduler(monkeypatch, ec_role="ec_producer")
req = _Request([_Feature("h1", length=1)])
@@ -378,8 +372,7 @@ def test_consumer_only_has_cache_item(monkeypatch):
def test_save_not_emitted_for_already_cached_entry(monkeypatch):
"""An entry already in the cache (from a prior save) must not trigger
a second allocation or appear in meta.saves again.
"""
a second allocation or appear in meta.saves again."""
s = _make_scheduler(monkeypatch)
req = _Request([_Feature("h1", length=1)])
@@ -400,8 +393,7 @@ def test_save_not_emitted_for_already_cached_entry(monkeypatch):
def test_update_connector_output_ignores_foreign_meta(monkeypatch):
"""A worker output whose payload is not an ECCPUWorkerMetadata is a
no-op — no cache mutation, no crash.
"""
no-op — no cache mutation, no crash."""
s = _make_scheduler(monkeypatch)
_seed_cached(s, "a", n_blocks=1)
@@ -415,8 +407,7 @@ def test_update_connector_output_ignores_foreign_meta(monkeypatch):
def test_update_connector_output_ignores_unknown_report(monkeypatch):
"""Reports for a never-seen save hash or a transfer id this scheduler never
dispatched are dropped rather than mutating unrelated state.
"""
dispatched are dropped rather than mutating unrelated state."""
s = _make_scheduler(monkeypatch)
s.update_connector_output(_WorkerOutput(saves=["gone"], loads=[4242]))
assert s.has_cache_item("gone") is False
@@ -428,8 +419,7 @@ def test_update_connector_output_ignores_unknown_report(monkeypatch):
def test_has_pending_push_work_tracks_inflight_save(monkeypatch):
"""A dispatched-but-unconfirmed save keeps push work pending until the
completion report marks it ready.
"""
completion report marks it ready."""
s = _make_scheduler(monkeypatch)
assert s.has_pending_push_work() is False
@@ -444,8 +434,7 @@ def test_has_pending_push_work_tracks_inflight_save(monkeypatch):
def test_has_pending_push_work_tracks_inflight_load(monkeypatch):
"""A dispatched-but-unconfirmed load keeps push work pending until the
unpin completion report arrives.
"""
unpin completion report arrives."""
s = _make_scheduler(monkeypatch)
_seed_cached(s, "a", n_blocks=1)
assert s.has_pending_push_work() is False # ready + unpinned
@@ -464,8 +453,7 @@ def test_has_pending_push_work_tracks_inflight_load(monkeypatch):
def test_shutdown_disables_roles_and_cleans_region(monkeypatch):
"""After shutdown the scheduler serves no items and the region is
cleaned up.
"""
cleaned up."""
s = _make_scheduler(monkeypatch, num_blocks=2)
_seed_cached(s, "a", n_blocks=2)
assert s.has_cache_item("a") is True
@@ -98,8 +98,7 @@ def test_second_instance_opens_existing_file_and_shares_memory():
def test_only_creator_unlinks_file_on_cleanup():
"""Critical contract: if the non-creator unlinks, the creator's mmap path
becomes a dangling backing file and a third opener would create a new one
out from under the creator.
"""
out from under the creator."""
instance_id = str(uuid.uuid4())
r1 = ECSharedRegion(
engine_id=instance_id,
@@ -216,8 +215,7 @@ def test_madvise_einval_selects_fallback_for_whole_region(monkeypatch):
def test_non_creator_does_not_probe_or_populate(monkeypatch):
"""Only the creator pre-faults. A second opener must not probe the advice
nor re-touch pages the creator already populated.
"""
nor re-touch pages the creator already populated."""
from vllm.distributed.ec_transfer.ec_connector.cpu import ec_shared_region as esr
instance_id = str(uuid.uuid4())
@@ -314,8 +312,7 @@ def test_madvise_unexpected_oserror_propagates(monkeypatch):
def test_pin_memory_success_sets_flag(region):
"""When cudaHostRegister returns 0, _is_pinned flips to True
and cleanup will correspondingly call cudaHostUnregister.
"""
and cleanup will correspondingly call cudaHostUnregister."""
fake_cudart = MagicMock()
success = MagicMock()
success.value = 0
@@ -335,8 +332,7 @@ def test_pin_memory_success_sets_flag(region):
def test_pin_memory_failure_leaves_flag_false():
"""If cudaHostRegister fails (non-zero), don't pretend it succeeded —
cleanup must NOT call cudaHostUnregister on memory we never registered.
"""
cleanup must NOT call cudaHostUnregister on memory we never registered."""
r = _make_region()
try:
fake_cudart = MagicMock()
@@ -363,8 +363,7 @@ def test_save_caches_coalesces_only_adjacent_blocks(
def test_save_caches_noop_when_mm_hash_not_in_saves(make_worker):
"""When the scheduler hasn't pre-allocated blocks for ``mm_hash``,
``save_caches`` + ``flush_saves`` is a pure no-op.
"""
``save_caches`` + ``flush_saves`` is a pure no-op."""
worker = make_worker()
sentinel = 0x42
worker._region.blocks.fill_(sentinel)
@@ -382,8 +381,7 @@ def test_save_caches_noop_when_mm_hash_not_in_saves(make_worker):
)
def test_save_caches_noop_for_non_save_rank(make_worker, tp_rank, pcp_rank):
"""Only TP rank 0 + PCP rank 0 writes to mmap. All other combinations
must skip the save entirely.
"""
must skip the save entirely."""
worker = make_worker(tp_rank=tp_rank, pcp_rank=pcp_rank)
sentinel = 0x42
worker._region.blocks.fill_(sentinel)
@@ -398,8 +396,7 @@ def test_save_caches_noop_for_non_save_rank(make_worker, tp_rank, pcp_rank):
def test_save_caches_raises_when_allocated_blocks_too_small(make_worker):
"""``save_caches`` must raise ``AssertionError`` when the encoder output
is larger than the allocated block space.
"""
is larger than the allocated block space."""
worker = make_worker()
src = torch.zeros(3 * _HIDDEN_DIM, dtype=_DTYPE)
with pytest.raises(AssertionError, match="exceeds allocated blocks"):
@@ -552,8 +549,7 @@ def test_start_load_caches_noop_when_loads_is_empty(make_worker):
def test_start_load_caches_skips_cached_and_loads_new_in_same_step(make_worker):
"""Every hash in ``meta.loads`` is copied from mmap, including one whose
key is already resident in ``encoder_cache`` — there is no worker-side
skip.
"""
skip."""
worker = make_worker()
n_blocks = 3
src_orig = torch.arange(n_blocks * _HIDDEN_DIM, dtype=_DTYPE).reshape(
@@ -782,8 +778,7 @@ def test_buffer_pool_is_reused_across_save_steps(make_worker):
@_requires_accelerator
def test_buffer_pool_is_reused_across_load_steps(make_worker):
"""Once a load copy completes its descriptor buffers return to the pool and
are reused by the next call — no reallocation.
"""
are reused by the next call — no reallocation."""
worker = make_worker()
worker._region.blocks[0].fill_(0x01)
worker._region.blocks[1].fill_(0x02)
@@ -854,8 +849,7 @@ def test_init_pins_memory_when_available(pin_available, expected_pinned):
def test_shutdown_calls_region_cleanup_and_swallows_errors(caplog_vllm):
"""``shutdown`` must always call ``region.cleanup`` — and must never
raise.
"""
raise."""
worker = object.__new__(ECCPUWorker)
mock_region = Mock(spec=ECSharedRegion)
worker._region = mock_region
@@ -43,8 +43,7 @@ def assign(proxy):
def _drive(assign, e_urls, counts):
"""Feed a sequence of request item-counts through the cursor, as
fanout_encoder_primer does one request at a time.
"""
fanout_encoder_primer does one request at a time."""
cursor = 0
all_urls = []
for count in counts:
+1 -2
View File
@@ -40,8 +40,7 @@ def test_metadata_saves_dict_operations():
def test_metadata_loads_carry_a_transfer_id_and_blocks():
"""Each load entry is a (transfer_id, block_ids) pair, so the worker can
report the transfer rather than the mm_hash.
"""
report the transfer rather than the mm_hash."""
meta = ECCPUConnectorMetadata()
meta.loads["mm_hash_1"] = (0, [5, 6, 7])
@@ -79,8 +79,7 @@ def test_request_finished_announces_not_ready_entry(monkeypatch):
"""A save whose GPU->mmap copy hasn't been confirmed complete yet is
still announced: the entry can't be evicted before it's ready, and a read
arriving too early is NACKed NACK_NOT_READY for the consumer to retry —
protecting it is not this method's job.
"""
protecting it is not this method's job."""
s = _sched_gate_off(monkeypatch)
s._nixl_enabled = True
s._peer_host, s._peer_port = "1.2.3.4", 5601
+2 -4
View File
@@ -129,8 +129,7 @@ def test_producer_session_grant_nacks_missing_entry():
def test_producer_session_grant_nacks_not_ready_entry_as_retryable():
"""An announced entry whose save is still in flight is distinguished from a
miss, so the consumer retries instead of falling back to local compute.
"""
miss, so the consumer retries instead of falling back to local compute."""
cache = EmbeddingCache(num_blocks=8)
cache.alloc("h1", 2) # not ready
s = _make_producer_session(cache)
@@ -438,8 +437,7 @@ def test_consumer_session_poll_nack_goes_to_tombstoned():
def test_consumer_session_poll_not_ready_nack_goes_to_retryable():
"""A not-ready NACK must not tombstone: tombstoning admits the request with
no embedding, which is fatal once the media has been rewritten away.
"""
no embedding, which is fatal once the media has been rewritten away."""
import msgspec
s = _make_consumer_session()
@@ -68,8 +68,7 @@ class _StubEngineCoreActor(EngineCoreActorMixin):
# in ``tests/entrypoints/test_api_server_process_manager.py``.
def _bind_and_report_worker(listen_address, sock, args, client_config):
"""Bind ROUTER/PULL with a kernel-assigned port, report the actual
endpoints back via ``actual_address_pipe``, then exit.
"""
endpoints back via ``actual_address_pipe``, then exit."""
ctx = zmq.Context()
try:
in_sock = make_zmq_socket(
+4 -8
View File
@@ -607,8 +607,7 @@ def _cadenced_dp_engine_core(
monkeypatch, results: list[tuple[bool, bool]], dp_sync_interval: int = 32
):
"""A bare DPEngineCoreProc whose all-reduce is scripted by `results`;
returns the core and the step numbers at which the all-reduce ran.
"""
returns the core and the step numbers at which the all-reduce ran."""
core = object.__new__(DPEngineCoreProc)
core.dp_group = object()
core.dp_sync_interval = dp_sync_interval
@@ -627,8 +626,7 @@ def _cadenced_dp_engine_core(
def test_dp_sync_interval_default_is_16():
"""Regression pin: lowering the default narrows the mid-wave pause tail
for async RL, where the engine is rarely idle when pause lands.
"""
for async RL, where the engine is rarely idle when pause lands."""
assert ParallelConfig.dp_sync_interval == 16
@@ -646,8 +644,7 @@ def test_dp_sync_interval_normal_wave(monkeypatch, dp_sync_interval: int):
def test_dp_sync_interval_idle_pause_consensus_on_first_step(monkeypatch):
"""A pause of an idle engine arms every rank before its kick-started
first step, so the step-1 sync reaches consensus after one dummy batch
regardless of the configured cadence.
"""
regardless of the configured cadence."""
core, synced = _cadenced_dp_engine_core(
monkeypatch, [(False, True)], dp_sync_interval=32
)
@@ -673,8 +670,7 @@ def _pausable_engine_core_proc() -> EngineCoreProc:
@pytest.mark.parametrize("deferred", [False, True])
def test_pause_synchronizes_device_before_cache_reset(deferred: bool):
"""A resolved pause promises an idle device: the barrier must run before
caches are cleared and before the caller is unblocked.
"""
caches are cleared and before the caller is unblocked."""
core = _pausable_engine_core_proc()
core.engines_running = deferred
order: list[str] = []
+3 -6
View File
@@ -249,8 +249,7 @@ def test_dplb_non_late_interaction_still_uses_lb():
def test_dplb_burst_round_robins_despite_snapshot_rebinds():
"""A stats snapshot rebind wipes the optimistic lb_engines increments;
the exact in-flight floor must keep a burst spreading round-robin.
"""
the exact in-flight floor must keep a burst spreading round-robin."""
client = _make_dplb_client(num_engines=4)
for _ in range(4):
@@ -325,8 +324,7 @@ async def test_dplb_scale_down_routes_after_stale_stats_snapshot():
def test_dplb_snapshot_backpressure_overrides_inflight():
"""An engine reported heavily loaded by the coordinator is avoided even
when this client has routed nothing to it.
"""
when this client has routed nothing to it."""
client = _make_dplb_client(num_engines=2)
client.lb_engines = [[5, 10, 0.0], [0, 0, 0.0]]
@@ -339,8 +337,7 @@ def test_dplb_snapshot_backpressure_overrides_inflight():
def test_dplb_kv_pressure_amplifies_waiting_penalty():
"""A waiting queue on a KV-bound engine (slow drain) is penalized, while
the same queue with low KV usage is not (e.g. transient burst).
"""
the same queue with low KV usage is not (e.g. transient burst)."""
client = _make_dplb_client(num_engines=2)
# Engine 0 has a smaller total but is KV-bound with a queue.
client.lb_engines = [[5, 10, 1.0], [0, 20, 0.2]]
+1 -2
View File
@@ -30,8 +30,7 @@ def _make_processor(num_logprobs: int) -> LogprobsProcessor:
def test_drops_trailing_sentinel_columns():
"""A request that asked for 3 custom token logprobs but ended up in a
batch padded to width 5 must not surface the trailing -inf entries.
"""
batch padded to width 5 must not surface the trailing -inf entries."""
processor = _make_processor(num_logprobs=3)
sampled = 42
+1 -2
View File
@@ -229,8 +229,7 @@ def test_request_stream_interval_raises_but_not_below_engine_default(
):
"""A per-request stream_interval can raise the interval above the engine
default but not below it (values under the default clamp up), without
altering the generated text.
"""
altering the generated text."""
engine_stream_interval = 5
# Request 0 (below the default) clamps up to 5; request 1 raises it to 10.
request_stream_intervals = [1, 10]
@@ -80,8 +80,7 @@ def _split_hnd_cache(spec) -> torch.Tensor:
def _packed_nhd_cache(spec) -> torch.Tensor:
"""Logical (num_blocks, heads, block_size, 2 * head_size) over an NHD
physical layout — the FlashAttention/FlashInfer/Triton/Flex form.
"""
physical layout — the FlashAttention/FlashInfer/Triton/Flex form."""
return torch.zeros(
NUM_BLOCKS,
spec.block_size,
@@ -863,8 +863,7 @@ def test_parallelism_agnostic_excluded(kv_cache_groups: list[KVCacheGroupSpec]):
)
def test_canonical_layout_gate(kv_cache_groups, certified):
"""The canonical layout certifies portability group by group; none of
these shapes are portable in the direct layout.
"""
these shapes are portable in the direct layout."""
assert not _parallelism_agnostic(kv_cache_groups)
assert _parallelism_agnostic(kv_cache_groups, canonical=True) is certified
@@ -872,8 +871,7 @@ def test_canonical_layout_gate(kv_cache_groups, certified):
def test_canonical_layout_certifies_v2_model_runner():
"""Canonical bytes are certified per layer against live tensor strides at
registration, so the static gate must not depend on the model-runner
version — the v2 runner is the case the canonical layout exists for.
"""
version — the v2 runner is the case the canonical layout exists for."""
groups = _groups(_full_attention_spec())
assert _parallelism_agnostic(groups, canonical=True, v2=True)
@@ -117,8 +117,7 @@ def _spec_cls_with_metric_definitions(
metric_definitions: dict[str, Any],
) -> type:
"""Build a fake offloading spec class reporting the given metric
definitions, so tests don't need to patch the real CPU spec.
"""
definitions, so tests don't need to patch the real CPU spec."""
class _FakeOffloadingSpec:
@staticmethod
@@ -188,8 +187,7 @@ def test_build_kv_connector_stats_with_empty_dict():
def test_build_kv_connector_stats_reconstructs_offload_stats():
"""Test that OffloadingConnector stats are properly reconstructed with
correct data.
"""
correct data."""
serialized_data = {
_StatsKey.TYPES: {
LOAD_BYTES: _MetricType.COUNTER,
@@ -1583,8 +1583,7 @@ _SCAN_BEHAVIOR = {
@pytest.mark.parametrize("result", list(LookupResult))
def test_scan_behavior_declared_for_every_lookup_result(result: LookupResult):
"""Whether a result keeps a sliding-window streak alive decides how wide
the demanded chunk set gets, so every member needs deliberate behavior.
"""
the demanded chunk set gets, so every member needs deliberate behavior."""
assert result in _SCAN_BEHAVIOR, f"{result} has no declared scan behavior"
expected_lookups, expected_end = _SCAN_BEHAVIOR[result]
@@ -2035,8 +2034,7 @@ def test_request_level_policy_stores_all_blocks(request_runner, async_scheduling
def test_loads_do_not_populate_fence_index(request_runner):
"""Loads don't populate _block_id_to_pending_jobs (protected by
delay_free_blocks while in flight).
"""
delay_free_blocks while in flight)."""
runner = request_runner(
blocks_per_chunk=3,
block_size=4,
@@ -2106,8 +2104,7 @@ def test_fence_at_update_state_after_alloc(request_runner):
def test_fence_at_build_store_jobs(request_runner):
"""A new prefill (no load -> update_state_after_alloc returns early)
reusing a finished request's pending-store block is flushed by
_build_store_jobs's fence.
"""
_build_store_jobs's fence."""
runner = request_runner(
blocks_per_chunk=1,
block_size=4,
@@ -2157,8 +2154,7 @@ def test_fence_at_build_store_jobs(request_runner):
@pytest.mark.parametrize("async_scheduling", [True, False])
def test_complete_store_called_per_job(request_runner, async_scheduling: bool):
"""complete_store fires per-job, not deferred to request finish.
Each call carries only that store's keys.
"""
Each call carries only that store's keys."""
tokens_per_block = 4
blocks_per_chunk = 3
tokens_per_chunk = tokens_per_block * blocks_per_chunk
@@ -2387,8 +2383,7 @@ def test_offload_prompt_only(request_runner, async_scheduling: bool):
@pytest.mark.parametrize("async_scheduling", [True, False])
def test_reset_cache(request_runner, async_scheduling: bool):
"""reset_cache flushes in-flight loads, calls manager.reset_cache(), resets
next_stored_chunk_idx for active requests and clears job tracking.
"""
next_stored_chunk_idx for active requests and clears job tracking."""
block_size = 4
blocks_per_chunk = 3
tokens_per_chunk = block_size * blocks_per_chunk
@@ -2934,8 +2929,7 @@ def test_stale_sliding_window_block_after_prepare_store_failure(
@pytest.mark.parametrize("async_scheduling", [True, False])
def test_skip_reading_prefix_cache(request_runner, async_scheduling: bool):
"""When skip_reading_prefix_cache=True, the offloading connector must not
load any blocks from CPU even if a matching prefix is cached there.
"""
load any blocks from CPU even if a matching prefix is cached there."""
block_size = 4
blocks_per_chunk = 3
tokens_per_chunk = block_size * blocks_per_chunk
@@ -3029,8 +3023,7 @@ def test_max_load_tokens_limits_external_load(request_runner, async_scheduling:
class TestEagle:
"""Tests for Eagle/MTP speculative decoding support in the offloading
connector scheduler — both _lookup() unit tests and integration tests.
"""
connector scheduler — both _lookup() unit tests and integration tests."""
# -------------------------------------------------------------------
# Helpers
@@ -3572,8 +3565,7 @@ class TestEagle:
self, request_runner, async_scheduling: bool
):
"""Eagle sliding-window group stores all prompt blocks but excludes
the trailing chunk while decoding.
"""
the trailing chunk while decoding."""
block_size = 4
sliding_window = 8
num_gpu_blocks = 100
@@ -3625,8 +3617,7 @@ class TestEagle:
self, request_runner, async_scheduling: bool
):
"""An eagle group with a single-block prompt stores it at the end of
prefill: prompt blocks are stable, so no tail is held back.
"""
prefill: prompt blocks are stable, so no tail is held back."""
block_size = 4
blocks_per_chunk = 1
tokens_per_chunk = block_size * blocks_per_chunk
@@ -4263,8 +4254,7 @@ def test_request_finished_mixed_full_attn_and_sliding_window(
def test_chunked_local_attention_reports_its_chunk_window():
"""Llama 4 uses chunked local attention, which used to trip the
FullAttentionSpec assert and kill the engine at startup.
"""
FullAttentionSpec assert and kill the engine at startup."""
spec = ChunkedLocalAttentionSpec(
block_size=16,
num_kv_heads=2,
@@ -4503,8 +4493,7 @@ def test_retention_interval_zero_stores_only_replay_boundary(
def _shared_kv_mtp_config():
"""Speculative config for a shared-group MTP model: eagle-family method
whose drafter layer merges into a target KV-cache group, so no group
self-identifies as a drafter group (issue #52735).
"""
self-identifies as a drafter group (issue #52735)."""
spec = MagicMock(name="shared_kv_mtp_spec")
spec.use_eagle.return_value = True
spec.use_eagle_block_drop.return_value = True
@@ -4518,14 +4507,12 @@ def _shared_kv_mtp_config():
class TestSharedGroupMTPOffload:
"""Regression tests for issue #52735: OffloadingConnector must keep
serving when speculative decoding is enabled but no KV-cache group is
annotated as a drafter group (shared-group MTP models).
"""
annotated as a drafter group (shared-group MTP models)."""
def test_no_annotation_marks_no_groups(self, request_runner):
"""Spec decode on + zero annotated groups must NOT mark every group
as a drafter group; the full store->load roundtrip must match the
non-speculative behavior of test_two_groups_full_and_sliding_window.
"""
non-speculative behavior of test_two_groups_full_and_sliding_window."""
block_size = 4
kv_cache_groups = [
KVCacheGroupSpec(
@@ -4582,8 +4569,7 @@ class TestSharedGroupMTPOffload:
class TestMambaHybridOffloadServing:
"""Store->finish->lookup flows on a full-attention + mamba-align hybrid,
with a manager that only HITs keys that were actually stored. Guards the
two collapse routes of issue #52735.
"""
two collapse routes of issue #52735."""
BLOCK = 4
MAMBA_BLOCK = 16
@@ -4712,8 +4698,7 @@ class TestMambaHybridOffloadServing:
"""Route 1 of #52735: with spec decode on and no drafter annotation,
serving must equal the non-speculative baseline (was 0 before the
fix: the all-groups fallback marked the mamba group as a drafter and
the volatile-tail pop consumed its only servable chunk).
"""
the volatile-tail pop consumed its only servable chunk)."""
baseline = self._roundtrip_served_tokens(self._make_scheduler(None))
assert baseline == 16
served = self._roundtrip_served_tokens(
@@ -4726,8 +4711,7 @@ class TestMambaHybridOffloadServing:
group must not drag the confirmed boundary below a coarser sibling's
chunk granularity. The widened query makes the volatile-tail pop
land on an extra queried chunk, holding the boundary at 16 tokens
(was 0 before the fix: 4 hits -> pop -> 12 tokens < mamba chunk).
"""
(was 0 before the fix: 4 hits -> pop -> 12 tokens < mamba chunk)."""
scheduler = self._make_scheduler(None, mamba_eagle=True)
assert [c.is_eagle_group for c in scheduler.config.kv_group_configs] == [
True,
@@ -80,8 +80,7 @@ def _allocate_kv_caches(
def _single_rank_vllm_config(total_kv_heads: int):
"""A one-rank (TP=1) parallel config, as canonical mappings are derived
from it.
"""
from it."""
vllm_config = MagicMock()
parallel_config = vllm_config.parallel_config
parallel_config.tensor_parallel_size = 1
@@ -148,8 +148,7 @@ def test_multiturn_lifecycle():
(do_remote_decode=True). Finishes LENGTH_CAPPED with remote_block_ids.
Turn 2: P receives remote_block_ids from D. P pulls KV from D because
remote_block_ids is not None and external tokens > 0. Computes only
new tokens, finishes LENGTH_CAPPED.
"""
new tokens, finishes LENGTH_CAPPED."""
vllm_config = create_vllm_config(
kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG,
)
@@ -195,8 +194,7 @@ def test_multiturn_lifecycle():
def test_first_turn_no_remote_blocks():
"""First turn: P has no remote_block_ids from D yet.
Standard local prefill, returns kv_transfer_params for future turns.
"""
Standard local prefill, returns kv_transfer_params for future turns."""
vllm_config = create_vllm_config(
kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG,
)
@@ -276,8 +274,7 @@ def test_abort_p_side_non_length_capped():
def test_remote_blocks_exceed_prompt_tokens():
"""D provides more remote tokens than P's prompt needs.
P caps external tokens to prompt length.
"""
P caps external tokens to prompt length."""
vllm_config = create_vllm_config(
kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG,
)
@@ -313,8 +310,7 @@ def test_remote_blocks_exceed_prompt_tokens():
def test_p_node_pulls_partial_last_block_from_d():
"""D sends remote_block_ids with partially filled last block.
remote_num_tokens < len(remote_block_ids) * block_size.
P pulls only remote_num_tokens worth of external tokens.
"""
P pulls only remote_num_tokens worth of external tokens."""
vllm_config = create_vllm_config(
kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG,
)
@@ -359,8 +355,7 @@ def test_p_node_pulls_partial_last_block_from_d():
def test_add_new_req_to_recv_populates_remote_meta():
"""add_new_req_to_recv correctly populates RemoteMeta for P-node
bi-directional KV pull from D.
"""
bi-directional KV pull from D."""
meta = NixlConnectorMetadata()
kv_params = {
"remote_block_ids": [[0, 1, 2]],
@@ -388,8 +383,7 @@ def test_add_new_req_to_recv_populates_remote_meta():
def test_build_connector_meta_recv_entries():
"""P-node scheduler: do_remote_decode=True + remote_block_ids →
_reqs_need_recv populated, build_connector_meta produces reqs_to_recv.
"""
_reqs_need_recv populated, build_connector_meta produces reqs_to_recv."""
vllm_config = create_vllm_config(
kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG,
)
@@ -463,8 +457,7 @@ def test_p_node_pull_kv_from_d(dist_init):
)
def test_p_node_pull_then_send_kv(dist_init):
"""Full P-node bi-directional: pull KV from D → prefill →
send KV back to D via notification.
"""
send KV back to D via notification."""
connector, worker = _make_connector_with_fake_worker()
meta = _make_p_node_recv_metadata("req-p2", [10, 11], [20, 21])
_do_load_kv(connector, meta)
@@ -511,8 +504,7 @@ def test_p_node_deferred_pull_on_no_handshake(dist_init):
def test_d_node_request_finished_returns_kv_params():
"""D-node request_finished returns kv_transfer_params with
do_remote_decode=True, remote_block_ids, remote_num_tokens
for P to pull. These params go directly to P node.
"""
for P to pull. These params go directly to P node."""
vllm_config = create_vllm_config(
kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG,
)
@@ -593,8 +585,7 @@ def test_d_node_request_finished_remote_num_tokens():
def test_d_node_partial_last_block_remote_num_tokens():
"""D-node: remote_num_tokens < len(remote_block_ids) * block_size
when last block is partially filled.
"""
when last block is partially filled."""
vllm_config = create_vllm_config(
kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG,
)
@@ -628,8 +619,7 @@ def test_no_double_read_blocks_after_reschedule():
bidirectional request (once on initial schedule, once after
WAITING_FOR_REMOTE_KVS → reschedule). The _remote_blocks_processed
flag must prevent the request from being added to _reqs_need_recv
twice, which would cause P to read D's blocks twice.
"""
twice, which would cause P to read D's blocks twice."""
vllm_config = create_vllm_config(
kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG,
)
@@ -682,8 +672,7 @@ def test_remote_num_tokens_bounded_by_blocks():
"""Edge case 2: D-node request_finished must return
remote_num_tokens <= len(remote_block_ids) * block_size.
request.num_tokens includes the last sampled token which has no KV
in the cache, so remote_num_tokens must use num_computed_tokens.
"""
in the cache, so remote_num_tokens must use num_computed_tokens."""
vllm_config = create_vllm_config(
kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG,
)
@@ -719,8 +708,7 @@ def test_remote_num_tokens_bounded_by_blocks():
def test_kv_recompute_threshold_skips_small_transfer():
"""Edge case 3: When remote tokens are below kv_recompute_threshold,
P should skip the remote pull and compute locally instead of
entering WAITING_FOR_REMOTE_KVS.
"""
entering WAITING_FOR_REMOTE_KVS."""
threshold = 256
vllm_config = create_vllm_config(
kv_connector_extra_config={
@@ -762,8 +750,7 @@ def test_p_node_finished_holds_blocks_for_d():
"""Edge case 4: P-node finishes with FINISHED_LENGTH_CAPPED and
do_remote_decode=True. P must hold blocks (delay_free=True) and
return kv_transfer_params with do_remote_prefill=True so D can
read P's blocks.
"""
read P's blocks."""
vllm_config = create_vllm_config(
kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG,
)
@@ -804,8 +791,7 @@ def test_p_node_finished_holds_blocks_for_d():
def test_cache_miss_first_turn_no_remote_pull():
"""Edge case 5: First turn with do_remote_decode=True but no
remote_block_ids (cache MISS). P should prefill locally with
num_external_tokens=0 and not enter WAITING_FOR_REMOTE_KVS.
"""
num_external_tokens=0 and not enter WAITING_FOR_REMOTE_KVS."""
vllm_config = create_vllm_config(
kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG,
)
@@ -840,8 +826,7 @@ def test_cache_miss_first_turn_no_remote_pull():
def test_partial_remote_tokens_less_than_prompt():
"""Edge case 6: D's remote_num_tokens covers only part of P's
prompt. P should pull remote_num_tokens worth of external tokens
and compute the rest locally.
"""
and compute the rest locally."""
vllm_config = create_vllm_config(
kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG,
)
@@ -886,8 +871,7 @@ def test_partial_remote_tokens_less_than_prompt():
def test_remote_blocks_processed_flag_persists():
"""Edge case 7: After recv completes and request is rescheduled,
the _remote_blocks_processed flag in kv_transfer_params prevents
the bidirectional path from re-entering _reqs_need_recv.
"""
the bidirectional path from re-entering _reqs_need_recv."""
vllm_config = create_vllm_config(
kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG,
)
@@ -959,8 +943,7 @@ _REMOTE = FakeNixlConnectorWorker.REMOTE_ENGINE_ID
)
def test_turn2_deadline_gate(dist_init, offset, expiry_delta, expect_declined):
"""P declines on the turn-2 readback if the deadline is known,
near-expiry and a handshake clock offset is known
"""
near-expiry and a handshake clock offset is known"""
connector, worker = _make_connector_with_fake_worker()
worker._engine_clock_offset[_REMOTE] = offset
expiry_time = None if expiry_delta is None else time.perf_counter() + expiry_delta
@@ -996,8 +979,7 @@ def test_turn2_deadline_gate(dist_init, offset, expiry_delta, expect_declined):
)
def test_turn2_full_prefix_hit_with_expired_deadline_skips_gate(dist_init):
"""A full local prefix hit issues no READ. The expiry gate must be skipped
even when D's deadline is expired.
"""
even when D's deadline is expired."""
connector, worker = _make_connector_with_fake_worker()
worker._engine_clock_offset[_REMOTE] = 0.0
@@ -1050,8 +1032,7 @@ def test_d_node_request_finished_exports_blocks_expiry_time():
def test_handshake_listener_appends_perf_counter_frame():
"""The handshake reply carries a 2nd frame with the listener's live
perf_counter.
"""
perf_counter."""
import threading
import msgspec
+6 -12
View File
@@ -19,15 +19,13 @@ class _StubLMCacheMPConnector:
test only asserts on the connector *name* and the ``extra_config`` dict
produced by ``VllmConfig``, never instantiates the connector, so a bare
placeholder class is sufficient. Not subclassing ``SupportsHMA`` mirrors
the real connector's HMA support (it does not support HMA either).
"""
the real connector's HMA support (it does not support HMA either)."""
@pytest.fixture
def stub_lmcache_mp_connector(monkeypatch):
"""Replace the lazy loader so VllmConfig.__post_init__ does not import
``lmcache_mp_connector`` (and thus ``lmcache``) during config tests.
"""
``lmcache_mp_connector`` (and thus ``lmcache``) during config tests."""
monkeypatch.setitem(
KVConnectorFactory._registry,
"LMCacheMPConnector",
@@ -107,8 +105,7 @@ def _build_config(
enable_cumem_allocator: bool = False,
) -> VllmConfig:
"""Build a VllmConfig that exercises _verify_kv_transfer_compat without
requiring a real model (avoids HF downloads in CI).
"""
requiring a real model (avoids HF downloads in CI)."""
from types import SimpleNamespace
kv_transfer_config = (
@@ -134,8 +131,7 @@ def test_kv_connector_rejects_expandable_segments(monkeypatch, kv_connector):
invalidated when expandable_segments lets the CUDA VMM allocator remap
the underlying physical pages. We can't enumerate every connector that
does this (especially OOT ones), so reject the combination whenever any
connector is configured.
"""
connector is configured."""
monkeypatch.setenv("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
with pytest.raises(ValueError, match="expandable_segments"):
_build_config(kv_connector=kv_connector)
@@ -143,8 +139,7 @@ def test_kv_connector_rejects_expandable_segments(monkeypatch, kv_connector):
def test_kv_connector_allows_expandable_segments_with_sleep_mode(monkeypatch):
"""Sleep mode routes KV allocations through CuMemAllocator's pool, which
auto-disables expandable_segments (see #40812).
"""
auto-disables expandable_segments (see #40812)."""
monkeypatch.setenv("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
_build_config(kv_connector="NixlConnector", enable_sleep_mode=True)
@@ -167,8 +162,7 @@ def test_kv_connector_allows_other_alloc_conf(monkeypatch):
def test_no_kv_connector_ignores_expandable_segments(monkeypatch):
"""The expandable_segments check only applies when a KV connector is
configured.
"""
configured."""
monkeypatch.setenv("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
_build_config(kv_connector=None)
@@ -50,8 +50,7 @@ def _make_flexkv_module(
impl_mock: MagicMock,
) -> tuple[types.ModuleType, types.ModuleType]:
"""Build a fake ``flexkv`` package hierarchy that returns *impl_mock*
when ``FlexKVConnectorV1Impl`` is instantiated.
"""
when ``FlexKVConnectorV1Impl`` is instantiated."""
flexkv_mod = types.ModuleType("flexkv")
integration_mod = types.ModuleType("flexkv.integration")
vllm_mod = types.ModuleType("flexkv.integration.vllm")
@@ -72,8 +71,7 @@ def _make_flexkv_module(
def _install_flexkv_mock(impl_mock: MagicMock):
"""Insert fake flexkv modules into sys.modules and return a context that
cleans them up afterwards.
"""
cleans them up afterwards."""
flexkv_mod, adapter_mod = _make_flexkv_module(impl_mock)
mods = {
"flexkv": flexkv_mod,
@@ -106,8 +104,7 @@ def _build_connector(vllm_config: VllmConfig, impl_mock: MagicMock):
class TestFlexKVConnectorImportError:
"""FlexKVConnectorV1 should fail with a helpful message when flexkv is
absent.
"""
absent."""
def test_import_error_message(self):
from vllm.distributed.kv_transfer.kv_connector.v1.flexkv_connector import (
@@ -176,8 +176,7 @@ def test_engine_unwraps_handshake_metadata_for_legacy_connector(
) -> None:
"""Engine core always asks workers for `(pp_rank, tp_rank)`-keyed metadata,
then unwraps to `{tp_rank: metadata}` for a connector that has not opted
into PP-aware handshake (single-PP producer, all `pp_rank == 0`).
"""
into PP-aware handshake (single-PP producer, all `pp_rank == 0`)."""
metadata_0 = _Metadata()
metadata_1 = _Metadata()
connector = _LegacyConnector()
@@ -200,8 +199,7 @@ def test_engine_rejects_pp_producer_for_legacy_connector(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""A connector that has not opted into PP-aware handshake must not silently
drop metadata from `pp_rank > 0`; engine core init raises instead.
"""
drop metadata from `pp_rank > 0`; engine core init raises instead."""
connector = _LegacyConnector()
with pytest.raises(ValueError, match="does not support PP-disaggregated"):
@@ -216,8 +214,7 @@ def test_engine_passes_handshake_metadata_through_for_pp_aware_connector(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""A PP-aware connector receives the full `(pp_rank, tp_rank)`-keyed dict
unchanged.
"""
unchanged."""
metadata_0 = _Metadata()
metadata_1 = _Metadata()
connector = _PPAwareConnector()
@@ -131,8 +131,7 @@ class TestHf3fsClientResourceManagement:
def test_close_idempotent_and_handles_cleared(self, tmp_path):
"""Multiple close() calls must not raise; deregister_fd called exactly
once, all handles set to None, shm.close() invoked.
"""
once, all handles set to None, shm.close() invoked."""
client, shm_r, shm_w = self._make_client(tmp_path)
with (
@@ -238,8 +237,7 @@ class TestHf3fsClientResourceManagement:
def test_constructor_failure_does_not_close_unallocated_shm(self, tmp_path):
"""Failure before SharedMemory is created must not raise AttributeError
or TypeError from cleanup.
"""
or TypeError from cleanup."""
with (
patch(f"{self._MOD}.HF3FS_AVAILABLE", True),
patch(f"{self._MOD}.register_fd"),
@@ -212,8 +212,7 @@ class TestUpdateConnectorOutput:
self, mock_connector
):
"""Test that method returns early when kv_cache_events is not
LMCacheKVEvents.
"""
LMCacheKVEvents."""
# Create a mock object that is not LMCacheKVEvents
fake_events = MagicMock()
connector_output = KVConnectorOutput(kv_cache_events=fake_events)
@@ -199,8 +199,7 @@ def _bare_worker() -> MooncakeConnectorWorker:
"""Construct a MooncakeConnectorWorker skipping __init__ (full init requires
a live TransferEngine). Only the attributes touched by the methods under
test are populated; role flags and async_zmq_ctx keep __del__'s shutdown
path a no-op.
"""
path a no-op."""
worker = MooncakeConnectorWorker.__new__(MooncakeConnectorWorker)
worker.xfer_stats = MooncakeKVConnectorStats()
worker.engine = MagicMock()
@@ -77,8 +77,7 @@ def test_store_tp_does_not_override_backend_kv_cache_layout(extra_config):
def _make_kv_cache_config() -> KVCacheConfig:
"""Single-group full-attention KVCacheConfig — enough for the connector
constructor's validate() pass.
"""
constructor's validate() pass."""
spec = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
return KVCacheConfig(
num_blocks=4,
@@ -693,8 +692,7 @@ def _poll_lookup(client, req_id, num_tokens=128, block_hashes=(), timeout=5.0):
def _gated_recv(gate: threading.Event, value: int):
"""Mock recv side-effect that blocks until ``gate`` is set, so the
executor's lookup can be held pending deterministically.
"""
executor's lookup can be held pending deterministically."""
def recv():
gate.wait()
@@ -705,8 +703,7 @@ def _gated_recv(gate: threading.Event, value: int):
def test_lookup_key_client_non_block_lookup_async():
"""Non-blocking lookup defers to the executor: None first, hit once the
Future resolves.
"""
Future resolves."""
vllm_config = _make_vllm_config()
with patch(
@@ -33,8 +33,7 @@ def _mamba_align(block_size=32):
def _make_coord(groups, hash_block_size, use_eagle=False, retention_interval=None):
"""Construct a coordinator using the natural LCM of group block sizes as
the scheduler block size — mirrors ``resolve_kv_cache_block_sizes`` for
the test fixtures.
"""
the test fixtures."""
block_sizes = [g.kv_cache_spec.block_size for g in groups]
scheduler_block_size = lcm(*block_sizes)
return MooncakeStoreCoordinator(
@@ -144,8 +143,7 @@ def test_coordinator_single_full_attention_no_hits():
def test_coordinator_single_swa_tautological_pool_masks_pre_window():
"""SWA tautological-pool: hit_length spans full prefix, mask is
tail-window only.
"""
tail-window only."""
groups = [KVCacheGroupSpec(["L0"], _swa(block_size=16, sliding_window=32))]
coord = _make_coord(groups, hash_block_size=16)
hs = _hashes(4) # 4 chunks * 16 tokens
@@ -192,8 +190,7 @@ def test_coordinator_hybrid_hole_in_full_clips_both():
def test_coordinator_group_block_size_double_hash():
"""Group block_size=32 over hash_block_size=16 hashes: adjacent
hashes merge before pool lookup.
"""
hashes merge before pool lookup."""
groups = [
KVCacheGroupSpec(["L0"], _full(16)),
KVCacheGroupSpec(["L1"], _full(32)),
@@ -217,8 +214,7 @@ def test_coordinator_group_block_size_double_hash():
def test_coordinator_fine_grained_partial_tail_hit():
"""K3 shape: FA + mamba-align, block_size=32 over hash_block_size=16. When
both groups have the sub-block boundary hash, the reconciled hit lands on
the hash boundary (48), not the block boundary (32).
"""
the hash boundary (48), not the block boundary (32)."""
groups = [
KVCacheGroupSpec(["L0"], _full(32)),
KVCacheGroupSpec(["L1"], _mamba_align(32)),
@@ -238,8 +234,7 @@ def test_coordinator_fine_grained_partial_tail_hit():
def test_coordinator_fine_grained_clips_when_one_group_missing_tail():
"""If only one group has the sub-block boundary, min-convergence clips the
reconciled hit back to the block boundary (32).
"""
reconciled hit back to the block boundary (32)."""
groups = [
KVCacheGroupSpec(["L0"], _full(32)),
KVCacheGroupSpec(["L1"], _mamba_align(32)),
@@ -280,8 +275,7 @@ def test_store_mask_swa_only_window_around_each_lcm_boundary():
"""Hybrid full-attn(block=32) + SWA(block=8, sw=8). lcm=32. With
aligned=64 the SWA group should mark exactly the blocks ending at 32
and 64 (i.e. blocks 3 and 7 at block_size=8); the rest can never
participate in any future hit.
"""
participate in any future hit."""
full = _full(32)
swa = _swa(block_size=8, sliding_window=8)
groups = [KVCacheGroupSpec(["L0"], full), KVCacheGroupSpec(["L1"], swa)]
@@ -327,8 +321,7 @@ def test_store_mask_swa_does_not_double_scale_under_dcp():
def test_store_mask_swa_wider_window_covers_more_blocks_per_lcm():
"""Same hybrid layout but sliding_window=16 (= 2 SWA blocks). Each lcm
boundary should now span two SWA tail blocks.
"""
boundary should now span two SWA tail blocks."""
full = _full(32)
swa = _swa(block_size=8, sliding_window=16)
groups = [KVCacheGroupSpec(["L0"], full), KVCacheGroupSpec(["L1"], swa)]
@@ -358,8 +351,7 @@ def test_store_mask_dsv4_5_groups_full_mla_plus_4_swa():
"""DSV4-shaped: full-MLA(B=256) + 4 SWA groups with B in {64, 64, 4, 8}
and varied sliding windows. lcm=256, hash_block_size=4. Two lcm segments
(aligned_len=512). Validates that the tile-once strategy produces the
expected per-segment tail-window pattern, repeated.
"""
expected per-segment tail-window pattern, repeated."""
full_mla = _full(block_size=256)
swa_64_sw128 = _swa(block_size=64, sliding_window=128)
swa_64_sw512 = _swa(block_size=64, sliding_window=512)
@@ -393,8 +385,7 @@ def test_store_mask_dsv4_5_groups_full_mla_plus_4_swa():
def test_store_mask_fast_path_all_block_sizes_equal_lcm():
"""When every non-full-attn group already aligns to lcm_block_size, the
fast path returns all-True without invoking find_longest_cache_hit.
"""
fast path returns all-True without invoking find_longest_cache_hit."""
full = _full(block_size=64)
swa = _swa(block_size=64, sliding_window=128)
groups = [KVCacheGroupSpec(["L0"], full), KVCacheGroupSpec(["L1"], swa)]
@@ -407,8 +398,7 @@ def test_store_mask_fast_path_all_block_sizes_equal_lcm():
def test_store_mask_fast_path_single_attention_group():
"""Two groups sharing the same SWA spec collapse to one attention group;
no lcm filter applies, every chunk is True.
"""
no lcm filter applies, every chunk is True."""
swa = _swa(block_size=16, sliding_window=32)
groups = [KVCacheGroupSpec(["L0"], swa), KVCacheGroupSpec(["L1"], swa)]
coord = _make_coord(groups, hash_block_size=16)
@@ -422,8 +412,7 @@ def test_store_mask_fast_path_single_attention_group():
def _retention_groups():
"""Hybrid full-attn(block=32) + SWA(block=8, sw=8); lcm=32. The SWA group
densely keeps one tail block per 32-token boundary.
"""
densely keeps one tail block per 32-token boundary."""
full = _full(32)
swa = _swa(block_size=8, sliding_window=8)
return [KVCacheGroupSpec(["L0"], full), KVCacheGroupSpec(["L1"], swa)]
@@ -431,8 +420,7 @@ def _retention_groups():
def test_store_mask_dense_default_matches_every_lcm_boundary():
"""retention_interval=None (default) keeps the SWA tail at every lcm
boundary: tokens 32/64/96/128 -> chunks 3/7/11/15.
"""
boundary: tokens 32/64/96/128 -> chunks 3/7/11/15."""
coord = _make_coord(_retention_groups(), hash_block_size=8)
masks = coord.store_mask(128)
assert masks[0] is None
@@ -442,8 +430,7 @@ def test_store_mask_dense_default_matches_every_lcm_boundary():
def test_store_mask_retention_interval_sparsifies_swa_tails():
"""retention_interval=64 keeps an SWA tail once per 64-token segment
(chunks 7 and 15) instead of every 32 tokens, dropping the mid-segment
boundaries at 32 and 96.
"""
boundaries at 32 and 96."""
coord = _make_coord(_retention_groups(), hash_block_size=8, retention_interval=64)
masks = coord.store_mask(128)
assert masks[0] is None # full attn unaffected
@@ -452,8 +439,7 @@ def test_store_mask_retention_interval_sparsifies_swa_tails():
def test_store_mask_retention_interval_zero_keeps_only_replay_boundary():
"""retention_interval=0 drops all segment tails; only the latest replay
boundary (capped at num_prompt-1, aligned down to lcm) is retained.
"""
boundary (capped at num_prompt-1, aligned down to lcm) is retained."""
coord = _make_coord(_retention_groups(), hash_block_size=8, retention_interval=0)
# No replay info -> nothing reachable for the SWA group.
assert coord.store_mask(128)[1] == [False] * 16
@@ -464,8 +450,7 @@ def test_store_mask_retention_interval_zero_keeps_only_replay_boundary():
def test_store_mask_retention_interval_keeps_segment_and_replay_tails():
"""Sparse segment tails (interval=64 -> chunks 7,15) plus the replay
boundary tail (num_prompt=100 -> chunk 11) coexist.
"""
boundary tail (num_prompt=100 -> chunk 11) coexist."""
coord = _make_coord(_retention_groups(), hash_block_size=8, retention_interval=64)
masks = coord.store_mask(128, num_prompt_tokens=100)
assert masks[1] == [i in (7, 11, 15) for i in range(16)]
@@ -503,8 +488,7 @@ def test_store_mask_excludes_mamba_groups_lookup_unaffected():
nulled/freed; speculative blocks relocate), so the positional normal save
must never cover mamba chunks — regardless of retention. Lookups still
probe mamba boundaries: their keys come from the pinned snapshot/CoW
hand-off path instead.
"""
hand-off path instead."""
groups = [
KVCacheGroupSpec(["L0"], _full(16)),
KVCacheGroupSpec(["L1"], _mamba_align(16)),
@@ -525,8 +509,7 @@ def test_store_mask_excludes_mamba_groups_lookup_unaffected():
def test_lookup_with_eagle_pops_last_full_attention_block():
"""Sanity: with use_eagle, find_longest_cache_hit drops the last block.
Pairs with the load_mask test below to lock the round-trip contract.
"""
Pairs with the load_mask test below to lock the round-trip contract."""
groups = [KVCacheGroupSpec(["L0"], _full(16))]
coord = _make_coord(groups, hash_block_size=16, use_eagle=True)
hs = _hashes(4)
@@ -567,8 +550,7 @@ def test_load_mask_with_eagle_does_not_double_prune_full_attention():
def test_load_mask_with_eagle_hybrid_full_plus_swa():
"""Hybrid (FullAttn + SWA) with eagle: load_mask must cover every chunk
in [0, token_len) for the FullAttn group; SWA group keeps its
tail-window mask.
"""
tail-window mask."""
groups = [
KVCacheGroupSpec(["L0"], _full(16)),
KVCacheGroupSpec(["L1"], _swa(16, 32)),
@@ -1,8 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""End-to-end save->lookup test for MooncakeStoreConnector on a hybrid
(SWA + Full) attention config, using a dict-backed mock store.
"""
(SWA + Full) attention config, using a dict-backed mock store."""
import sys
import threading
@@ -283,8 +282,7 @@ def test_e2e_swa_plus_full_save_then_lookup_hits():
def test_recv_skips_swa_blocks_before_window():
"""Producer stored every block for both groups; consumer must only fetch
SWA blocks within the sliding window, not the head.
"""
SWA blocks within the sliding window, not the head."""
full = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
# sliding_window=32, block_size=16 → 2 contiguous blocks within window.
swa = SlidingWindowSpec(
@@ -352,8 +350,7 @@ def test_recv_skips_swa_blocks_before_window():
def test_chunked_token_database_hash_block_size_smaller_than_block_size():
"""DSv4-style: hash_block_size=4, group block_size=16 — process_tokens
keys each chunk by its ending fine hash, including a partial tail.
"""
keys each chunk by its ending fine hash, including a partial tail."""
md = KeyMetadata("m", 0, 0, 0, 0, group_id=3)
db = ChunkedTokenDatabase(md, block_size=16, hash_block_size=4)
db.set_kv_caches_base_addr([0])
@@ -386,8 +383,7 @@ def test_sub_block_partial_tail_offload_reads_cow_block():
"""Sub-block prompt (the 900/128/1536 shape, scaled to 12/4/16): the
partial tail is offloaded for both groups under the boundary sub-hash. The
full-attention block is read from the request block table; the mamba block
is the core-provided CoW target, not block_ids.
"""
is the core-provided CoW target, not block_ids."""
full = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
mamba = MambaSpec(
block_size=16,
@@ -466,8 +462,7 @@ def test_sub_block_partial_tail_offload_reads_cow_block():
def test_offload_syncs_event_before_put():
"""An offload-carrying meta synchronizes its CoW-fence event before the
store put reads the blocks, then completes in one pass and drains the
completion counter.
"""
completion counter."""
full = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
mamba = MambaSpec(
block_size=16,
@@ -538,8 +533,7 @@ def test_sub_block_partial_tail_offload_covers_smaller_group_blocks():
offload must persist every FA block up to the boundary — the normal save
floors to the lcm, so those blocks are otherwise never written and the
consumer's per-group lookup would miss. The mamba boundary block still
reads the core-provided CoW target.
"""
reads the core-provided CoW target."""
full = FullAttentionSpec(block_size=4, num_kv_heads=8, head_size=64, dtype=None)
mamba = MambaSpec(
block_size=16,
@@ -810,8 +804,7 @@ def test_ring_scratch_group_is_never_stored_and_does_not_block_hits():
"""A per-request ring group (CircularBufferSpec, capacity as block_size)
sits beside the paged group: worker setup tolerates its block size (the
DeepSeek-V4.1 compressor ring is 8 rows), saving a request stores paged
blocks only, and lookup hits on the paged group alone.
"""
blocks only, and lookup hits on the paged group alone."""
full = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
ring = CircularBufferSpec(
block_size=8, num_kv_heads=1, head_size=64, head_size_v=0, dtype=torch.uint8
@@ -536,8 +536,7 @@ def _make_pending_load_unfinished_request(
def _make_pending_load_scheduler_output() -> SimpleNamespace:
"""scheduler_output for a step where req-0 is parked on a pending load
(not in scheduled_new_reqs or scheduled_cached_reqs).
"""
(not in scheduled_new_reqs or scheduled_cached_reqs)."""
return SimpleNamespace(
finished_req_ids=set(),
preempted_req_ids=set(),
@@ -713,8 +712,7 @@ def test_running_request_not_in_resumed_req_ids_appends_blocks():
def test_resumed_request_in_resumed_req_ids_replaces_blocks():
"""A request the scheduler marks resumed gets the FULL block table in
new_block_ids and must REPLACE the tracker's blocks (not append), even if
a stale tracker from before preemption is still present.
"""
a stale tracker from before preemption is still present."""
scheduler = _make_bare_scheduler()
_make_resumed_unfinished_request(
scheduler,
@@ -432,8 +432,7 @@ def _patch_worker_runtime(
def test_pool_key_to_string_without_prefix_is_unchanged():
"""Default (empty) cache_prefix keeps keys byte-identical to the
historical unprefixed format so existing deployments keep their hits.
"""
historical unprefixed format so existing deployments keep their hits."""
key = PoolKey(KeyMetadata("test-model", 0, 0, 0, 0), "deadbeef")
assert (
key.to_string() == "test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@deadbeef"
@@ -574,8 +573,7 @@ def test_tp_shared_receiving_reads_each_local_store_shard():
def test_pool_key_cache_prefix_namespaces_and_disambiguates():
"""A non-empty cache_prefix is prepended, and two instances with
different prefixes never collide on identical block hashes.
"""
different prefixes never collide on identical block hashes."""
md_a = KeyMetadata("test-model", 0, 0, 0, 0, cache_prefix="depA")
md_b = KeyMetadata("test-model", 0, 0, 0, 0, cache_prefix="depB")
@@ -591,8 +589,7 @@ def test_pool_key_cache_prefix_namespaces_and_disambiguates():
def test_default_local_buffer_size_matches_pr40900():
"""PR-40900 shipped a 4 GiB default for local_buffer_size; the dual-mode
patch preserves it (and the JSON key) so unchanged PR-40900 configs work.
"""
patch preserves it (and the JSON key) so unchanged PR-40900 configs work."""
assert worker.DEFAULT_LOCAL_BUFFER_SIZE == 4 * 1024**3
@@ -1014,8 +1011,7 @@ def test_store_sending_thread_skips_null_sparse_group_blocks():
def test_partial_tail_offload_skips_cap_omitted_mamba_group():
"""A Mamba group omitted by the handoff cap must not fall back to the
connector's positional block table.
"""
connector's positional block table."""
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheGroupSpec,
@@ -1143,8 +1139,7 @@ def test_partial_tail_put_failure_activates_pressure_gate():
def test_normal_save_excludes_mamba_group_and_null_blocks():
"""The positional normal save must never cover mamba chunks (align-mode
block tables are not append-only) and must skip chunks whose block is the
reserved null block, for any group.
"""
reserved null block, for any group."""
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheGroupSpec,
@@ -1205,8 +1200,7 @@ def test_normal_save_excludes_mamba_group_and_null_blocks():
def test_block_aligned_snapshot_offload_uses_provided_block():
"""A block-aligned hand-off (sparse-retention mamba checkpoint) uploads
exactly the core-pinned block under the boundary-end hash key; the
positional block table entry for that chunk must be ignored.
"""
positional block table entry for that chunk must be ignored."""
store = MagicMock()
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
store.batch_put_from_multi_buffers.side_effect = lambda keys, *a: [256] * len(keys)
@@ -1234,8 +1228,7 @@ def test_block_aligned_snapshot_offload_uses_provided_block():
def test_mixed_snapshot_and_sub_block_offloads():
"""A retention snapshot and the prompt-end sub-block CoW tail can arrive
in one hand-off; the sub-block path covers FA gap blocks but reads the
mamba boundary only from the CoW block, never positionally.
"""
mamba boundary only from the CoW block, never positionally."""
store = MagicMock()
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
store.batch_put_from_multi_buffers.side_effect = lambda keys, *a: [256] * len(keys)
@@ -1275,8 +1268,7 @@ def test_mixed_snapshot_and_sub_block_offloads():
def test_snapshot_offload_skips_null_handoff_block():
"""A hand-off the core could not materialize (null block) carries no
committed state; persisting it would poison the boundary key.
"""
committed state; persisting it would poison the boundary key."""
store = MagicMock()
thread = _make_partial_tail_send_thread(store)
@@ -1604,8 +1596,7 @@ def test_store_sending_thread_passes_replicate_config_when_preferred_segment_set
def test_store_sending_thread_passes_default_replicate_config_when_no_preferred_segment(): # noqa: E501
"""Without a preferred_segment the SendingThread still forwards a
(default-constructed) ReplicateConfig so the C++ side always sees a
well-defined config object.
"""
well-defined config object."""
store = MagicMock()
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
store.batch_put_from_multi_buffers.return_value = [256, 256]
@@ -2115,8 +2106,7 @@ def test_recv_thread_stops_after_first_failing_disk_offload_sub_batch():
def test_recv_thread_skips_split_when_budget_holds_all_keys():
"""PR-36 removed the count-based split trigger; with budget for 3 keys,
all three should be requested in a single call.
"""
all three should be requested in a single call."""
store = MagicMock()
store.batch_get_into_multi_buffers.return_value = [256, 256, 256]
thread = _make_store_recving_thread(
@@ -2330,8 +2320,7 @@ def test_requester_worker_init_skips_disk_budget_when_offload_disabled(
monkeypatch,
):
"""enable_offload=False zeroes out the disk budget so we don't generate
redundant owner GET-RPCs.
"""
redundant owner GET-RPCs."""
store = MagicMock()
store.setup.return_value = 0
_install_fake_mooncake(monkeypatch, store)
@@ -3006,8 +2995,7 @@ def test_requester_worker_group_semantics_string_true_enables(
def test_store_sending_thread_clamps_token_len_to_lcm():
"""Partial chunks past the last lcm boundary aren't stored — cache hits
are always lcm-aligned (mirrors HybridKVCacheCoordinator).
"""
are always lcm-aligned (mirrors HybridKVCacheCoordinator)."""
store = MagicMock()
store.batch_is_exist.return_value = [0, 0]
store.batch_put_from_multi_buffers.return_value = [256, 256]
@@ -3032,8 +3020,7 @@ def test_store_sending_thread_clamps_token_len_to_lcm():
def test_store_sending_thread_skips_when_token_len_below_lcm():
"""Requests shorter than lcm_block_size cannot produce any aligned chunk,
so neither the existence check nor the put should be issued.
"""
so neither the existence check nor the put should be issued."""
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheGroupSpec
store = MagicMock()
@@ -3823,8 +3810,7 @@ def test_lookup_full_hit_with_eagle_pops_once_not_twice():
"""Eagle already leaves the last block for the drafter, so a
full-prompt re-derivation must never fire for eagle-governed hits:
firing would anchor the search one block lower and pop a second
block, regressing the hit by an extra producer boundary.
"""
block, regressing the hit by an extra producer boundary."""
worker = _make_bare_worker(block_size=16)
worker.coord = mooncake_store_worker.MooncakeStoreCoordinator(
worker._kv_cache_groups,
@@ -3981,8 +3967,7 @@ def test_lookup_full_hit_swa_degrades_when_no_stored_boundary_is_usable():
the full hit into 48, whose SWA window needs the never-written block 1,
so every load failed and the recompute re-entered the same lookup. The
re-derivation must report that no stored boundary below the request end
is usable.
"""
is usable."""
from vllm.v1.kv_cache_interface import KVCacheGroupSpec, SlidingWindowSpec
worker = _make_bare_worker(block_size=16)
@@ -4004,8 +3989,7 @@ def test_lookup_full_hit_swa_degrades_when_no_stored_boundary_is_usable():
def test_lookup_swa_single_group_returns_full_when_tail_window_present():
"""Single-SWA, sliding_window=32 (= 2 blocks): producer stored only the
tail. Coordinator-driven lookup returns full prefix even though the
pre-window blocks are absent.
"""
pre-window blocks are absent."""
from vllm.v1.kv_cache_interface import KVCacheGroupSpec, SlidingWindowSpec
worker = _make_bare_worker(block_size=16)
@@ -4401,8 +4385,7 @@ def test_config_defaults_to_embedded():
def test_config_pr40900_unchanged(tmp_path):
"""A literal PR-40900 config (no mode, no enable_offload, no preferred_segment)
parses without raising and resolves to embedded mode.
"""
parses without raising and resolves to embedded mode."""
config_path = _write_mooncake_config(
tmp_path,
{
@@ -4510,8 +4493,7 @@ def test_config_zero_local_buffer():
def test_topology_standalone_store_with_disk_offload(tmp_path, monkeypatch):
"""standalone-store + disk: global_segment_size=0, enable_offload=True,
preferred_segment set. Assert setup() positional args, ReplicateConfig
wiring, and that the disk-offload buffer budget is allocated.
"""
wiring, and that the disk-offload buffer budget is allocated."""
store = MagicMock()
store.setup.return_value = 0
fake_replicate_config_cls = _install_fake_mooncake(monkeypatch, store)
@@ -4560,8 +4542,7 @@ def test_topology_standalone_store_with_disk_offload(tmp_path, monkeypatch):
def test_topology_embedded_cpu_only(tmp_path, monkeypatch):
"""Embedded + CPU-only: no mode key (defaults to embedded),
global_segment_size>0, enable_offload absent, no preferred_segment.
This is the PR-40900 baseline recipe.
"""
This is the PR-40900 baseline recipe."""
store = MagicMock()
store.setup.return_value = 0
fake_replicate_config_cls = _install_fake_mooncake(monkeypatch, store)
@@ -4750,8 +4731,7 @@ def test_store_worker_close_swallows_store_errors():
def test_blob_block_hashes_wire_roundtrip():
"""The lookup wire format sends a ``hash_len`` frame plus the raw hashes
concatenated back-to-back; the server rebuilds them through a zero-copy
``BlobBlockHashes`` view over the frame buffer.
"""
``BlobBlockHashes`` view over the frame buffer."""
hashes = [BlockHash(bytes([i]) * 16) for i in range(5)]
hash_len = len(hashes[0])
@@ -692,8 +692,7 @@ def test_moriio_handshake_returns_metadata(mock_parallel_groups):
def test_resolve_host_ip_prefers_extra_config():
"""An explicit ``host_ip`` in kv_connector_extra_config overrides get_ip()
(so an external router can advertise a routable/internal address); an
absent or empty value falls back to get_ip().
"""
absent or empty value falls back to get_ip()."""
assert resolve_host_ip({"host_ip": "10.0.0.7"}) == "10.0.0.7"
fallback = get_ip()
@@ -747,8 +746,7 @@ def _read_scheduler(
def test_hma_blocks_per_sw_two_groups():
"""A Full + SlidingWindow config runs with HMA and computes block budgets
correctly
"""
correctly"""
scheduler = _read_scheduler(_make_hybrid_kv_cache_config())
assert scheduler._is_hma_required is True
# cdiv(32, 16) + 1 == 3 for the sliding-window group, 0 for full attention.
@@ -766,8 +764,7 @@ def test_hma_blocks_per_sw_two_groups():
def test_is_hma_required(swa_enabled, disable_hma, expected_is_hma):
"""_is_hma_required tracks both the KV cache groups and the
--disable-hybrid-kv-cache-manager flag. When HMA is off,
get_exchange_clipped_blocks must be a no-op.
"""
get_exchange_clipped_blocks must be a no-op."""
config = (
_make_hybrid_kv_cache_config() if swa_enabled else _make_test_kv_cache_config()
)
@@ -780,8 +777,7 @@ def test_is_hma_required(swa_enabled, disable_hma, expected_is_hma):
def test_non_sliding_window_hybrid_is_rejected():
"""A hybrid group that is not sliding-window (e.g. chunked-local
attention) must fail closed rather than be silently mistransferred.
"""
attention) must fail closed rather than be silently mistransferred."""
full_spec = FullAttentionSpec(
block_size=16, num_kv_heads=4, head_size=64, dtype=torch.float16
)
@@ -815,8 +811,7 @@ def test_non_sliding_window_hybrid_is_rejected():
def test_token_count_basis_uses_full_attention_group():
"""Chunked-prefill token counting must use an unclipped full-attention
group (blocks_per_sw == 0), not a clipped sliding-window group.
"""
group (blocks_per_sw == 0), not a clipped sliding-window group."""
scheduler = _read_scheduler(_make_hybrid_kv_cache_config())
# Group 0 is the full-attention group
assert scheduler._full_attn_group_idx == 0
@@ -825,8 +820,7 @@ def test_token_count_basis_uses_full_attention_group():
def test_get_exchange_clipped_blocks_clips_only_sw_group():
"""get_exchange_clipped_blocks keeps the full attn group intact and clips the
sliding-window group to its window tail.
"""
sliding-window group to its window tail."""
scheduler = _read_scheduler(_make_hybrid_kv_cache_config())
full = [10, 11, 12, 13, 14]
sw = [20, 21, 22, 23, 24]
@@ -837,8 +831,7 @@ def test_get_exchange_clipped_blocks_clips_only_sw_group():
def test_metadata_hma_block_ids_preserved_per_group():
"""add_new_req stores per-group (BlockIds) block lists unchanged for both
read and write, so the hybrid group structure is retained.
"""
read and write, so the hybrid group structure is retained."""
metadata = MoRIIOConnectorMetadata()
# Assume:
@@ -110,8 +110,7 @@ def make_meta(*, p_tp: int, p_dp: int, remote_dp_rank: int, host: str = "phost0"
def build_decode_workers(cfg: PDConfig) -> list:
"""Decode workers that issue reads for one prefill instance. A TP decode
instance reads from every tp rank; a DP decode instance from each dp-rank
worker. Reused across requests so per-worker round-robin state advances.
"""
worker. Reused across requests so per-worker round-robin state advances."""
if cfg.d_tp > 1:
return [
make_decode_worker(world_size=cfg.d_tp, tp_rank=r, dp_rank=0)
@@ -126,8 +125,7 @@ def prefill_target_multiset(cfg: PDConfig, rounds: int) -> Counter:
"""Drive the REAL _resolve_read_source over a fair input stream; tally which
prefill GPU each read targets. The only modelled assumption is the proxy
contract -- each owner dp-rank delivered equally (``for owner_dp in
range(p_dp)``); no proxy algorithm is reproduced.
"""
range(p_dp)``); no proxy algorithm is reproduced."""
workers = build_decode_workers(cfg)
hits: Counter = Counter()
for _ in range(rounds):
@@ -555,8 +555,7 @@ class TestMultiConnectorStats:
def test_build_kv_connector_stats_reconstructs_nixl_stats(self):
"""Test that NixlConnector stats are properly reconstructed with
correct data.
"""
correct data."""
serialized_data = {
"NixlConnector": {
"transfer_duration": [1.5, 2.3],
@@ -318,8 +318,7 @@ def test_abort_immediately_remote_prefill_enqueues_empty_recv():
"""A remote-prefill request added with abort_immediately=True should
be added to the scheduler's waiting queue then immediately aborted, so the
NIXL connector's request_finished hook enqueues an empty recv to notify
the prefill instance to free its blocks.
"""
the prefill instance to free its blocks."""
from vllm.v1.request import RequestStatus
scheduler = create_scheduler(create_vllm_config())
@@ -3146,8 +3145,7 @@ def test_split_read_failure_defers_report_until_last_handle(
request — nor invalidate its blocks — while the sibling xfer is still in
flight: a posted READ cannot be aborted and would DMA into blocks the
scheduler could free and reuse. The report happens exactly once, when the
last handle is terminal.
"""
last handle is terminal."""
request_id = "split_read_partial_failure"
err_handle, live_handle = 11, 22
connector, worker, wrapper = _make_split_read_connector(
@@ -281,8 +281,7 @@ def test_update_state_after_alloc_tracks_cached_blocks_per_group():
"""Hybrid SWA+FA requests can have different prefix-cache-hit counts per
KV cache group. Each group's count must be tracked independently rather
than collapsed to a single scalar, or DCP read-phase alignment
(local_num_computed_blocks) would misalign one of the groups.
"""
(local_num_computed_blocks) would misalign one of the groups."""
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.pull_scheduler import (
NixlPullConnectorScheduler,
)
@@ -336,8 +335,7 @@ def test_full_local_hit_is_not_awaited_by_the_scheduler():
owed to the producer -- so the worker must not report it in
finished_recving. Doing so trips `assert RequestStatus.is_finished` in
_update_from_kv_xfer_finished, which only tolerates a finished recv for a
request that is parked or already done.
"""
request that is parked or already done."""
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.pull_scheduler import (
NixlPullConnectorScheduler,
)
@@ -442,8 +440,7 @@ def test_apply_dcp_prefix_caching_matches_global_logical_positions(
"""_apply_dcp_prefix_caching must operate on logical block IDs (DCP
interleaves at logical-block granularity) and always return equal-length
local/remote slices, so a later, unconditional _apply_prefix_caching
trim is a guaranteed no-op rather than double-processing the read.
"""
trim is a guaranteed no-op rather than double-processing the read."""
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import (
NixlConnectorWorker,
)
@@ -842,8 +839,7 @@ def test_apply_prefix_caching_ssm_prefix_cache_hit(
def test_apply_prefix_caching_ssm_unpairable_slots_rejected():
"""Local SSM slots can only exceed the remote ones by the position D
recomputes itself. A larger excess means the lists aren't
position-aligned: fail loudly rather than transfer into wrong slots.
"""
position-aligned: fail loudly rather than transfer into wrong slots."""
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import (
NixlConnectorWorker,
)
@@ -1159,8 +1155,7 @@ def test_nixl_region_sort_preserves_pipeline_layer_order():
@pytest.mark.cpu_test
def test_get_block_descs_ids_hybrid_ssm():
"""Test _compute_desc_ids uses per-group strides for hybrid
FA+SSM when ratio=1 (no kernel block size mismatch).
"""
FA+SSM when ratio=1 (no kernel block size mismatch)."""
from vllm.v1.kv_cache_interface import FullAttentionSpec, MambaSpec
worker = _make_mock_worker_for_desc_ids(
@@ -1244,8 +1239,7 @@ def test_get_block_descs_ids_uses_per_region_pool_capacity():
@pytest.mark.cpu_test
def test_get_block_descs_ids_kernel_block_mismatch():
"""Test _compute_desc_ids uses different strides for FA
(kernel blocks) vs SSM (logical blocks) when ratio > 1.
"""
(kernel blocks) vs SSM (logical blocks) when ratio > 1."""
from vllm.v1.kv_cache_interface import FullAttentionSpec, MambaSpec
ratio = 4
@@ -1276,8 +1270,7 @@ def test_get_block_descs_ids_kernel_block_mismatch():
def test_get_block_descs_ids_hetero_block_size_hybrid():
"""With a block-size ratio, FA desc ids are ratio-expanded while SSM
desc ids keep the unexpanded logical stride (state blocks are never
sub-split).
"""
sub-split)."""
from vllm.v1.kv_cache_interface import FullAttentionSpec, MambaSpec
worker = _make_mock_worker_for_desc_ids(
@@ -1316,8 +1309,7 @@ def _bind_worker_method(worker, name):
@pytest.mark.cpu_test
def test_map_block_ids_for_block_size_ratio_hybrid():
"""Attention groups expand to remote granularity and clip to the remote
coverage; mamba state blocks pass through 1:1.
"""
coverage; mamba state blocks pass through 1:1."""
from unittest.mock import MagicMock
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import (
@@ -1348,8 +1340,7 @@ def test_map_block_ids_for_block_size_ratio_hybrid():
@pytest.mark.cpu_test
def test_post_process_zeroes_untransferred_tail():
"""The untransferred sub-blocks of the last local block are zeroed on
receive; mamba state caches are untouched by the attention permute.
"""
receive; mamba state caches are untouched by the attention permute."""
from unittest.mock import MagicMock
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import (
@@ -1392,8 +1383,7 @@ def test_post_process_zeroes_untransferred_tail():
@pytest.mark.cpu_test
def test_nixl_metadata_hybrid_ssm_block_ids():
"""Test NixlConnectorMetadata correctly stores block IDs for FA + SSM
groups with different block counts (kernel mismatch active).
"""
groups with different block counts (kernel mismatch active)."""
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
NixlConnectorMetadata,
)
@@ -1474,8 +1464,7 @@ def _make_fake_kv_cache_manager():
@pytest.mark.cpu_test
def test_zeroing_block_ids_cover_only_loaded_attention_blocks():
"""Only zero-recorded (attention) groups contribute, sliced to the
externally-loaded token range; Mamba state blocks are never zeroed.
"""
externally-loaded token range; Mamba state blocks are never zeroed."""
manager = _make_fake_kv_cache_manager()
# Tokens [0, 16) are locally cached; the load covers tokens [16, 56).
@@ -1503,8 +1492,7 @@ def test_scheduler_filters_connector_loaded_blocks_from_zeroing():
@pytest.mark.cpu_test
def test_failed_load_rezeroes_unwritten_skipped_blocks():
"""A failed async load leaves zeroing-skipped blocks unwritten beyond
the valid prefix; they must be zeroed before local recompute.
"""
the valid prefix; they must be zeroed before local recompute."""
from unittest.mock import MagicMock
from vllm.v1.core.sched.scheduler import Scheduler
@@ -1964,8 +1952,7 @@ def test_logical_to_kernel_block_ids_with_remote_ratio(
def test_exchange_clipped_blocks_ssm_single_state():
"""In single-state cache modes, SSM lists are reduced to the running
state slot: speculative scratch slots, null placeholders and the previous
step's state carry nothing. Attention groups pass through untouched.
"""
step's state carry nothing. Attention groups pass through untouched."""
sched = make_nixl_scheduler(has_mamba=True, is_hma_required=True)
sched.blocks_per_sw = [0, 0]
sched._ssm_spec_blocks = [None, 2]
@@ -1992,8 +1979,7 @@ def test_exchange_clipped_blocks_ssm_single_state():
@pytest.mark.cpu_test
def test_exchange_clipped_blocks_ssm_positional_states():
"""In "all" mode every position holds a state, so only the speculative
slots go; placeholders stay to keep the list position-indexed.
"""
slots go; placeholders stay to keep the list position-indexed."""
sched = make_nixl_scheduler(has_mamba=True, is_hma_required=True)
sched.blocks_per_sw = [0, 0]
sched._ssm_spec_blocks = [None, 2]
@@ -2010,8 +1996,7 @@ def _make_hybrid_mla_kv_cache_config(num_blocks: int = 4):
"""KimiLinear-shaped config: one MLA group and two KDA (GDN-typed
MambaSpec) groups whose layers share the same HMA tensors, with a
mamba-aligned unified page and an MLA kernel block smaller than the
logical block.
"""
logical block."""
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
from vllm.v1.kv_cache_interface import (
KVCacheConfig,
@@ -2151,8 +2136,7 @@ def test_register_kv_caches_hybrid_mla_dual_purpose_regions():
"""Hybrid MLA+KDA registration: HMA tensors shared by both layer types
must be flagged as MLA regions even when a KDA layer registers them
first, expose TP-independent kernel-granularity block lens, and build
FA + mamba descriptors for every region.
"""
FA + mamba descriptors for every region."""
from unittest.mock import MagicMock
from vllm.config import set_current_vllm_config
@@ -2253,8 +2237,7 @@ def test_register_kv_caches_hybrid_mla_dual_purpose_regions():
def test_push_write_hybrid_mla_replicates_attention():
"""Hybrid MLA+SSM push with P_TP < D_TP: attention blocks must be
written to every covered D rank (replicated MLA latent) while SSM state
is written per-rank through the split handles.
"""
is written per-rank through the split handles."""
import threading
from collections import defaultdict
from unittest.mock import MagicMock
@@ -24,8 +24,7 @@ from .utils import create_vllm_config
class _RecordingNixl:
"""Minimal NIXL wrapper stand-in that records descriptor lists and
prepared transfers so tests can resolve desc ids to byte ranges.
"""
prepared transfers so tests can resolve desc ids to byte ranges."""
def __init__(self, *args, **kwargs):
self.dlists: dict[int, np.ndarray] = {}
@@ -594,8 +593,7 @@ def _register_remote_agents(worker, metadata, tp_size):
def _owned_byte_ranges(worker, group_logical_ids):
"""Byte ranges owned by a request: for each HMA region tensor, every
logical block id of every group maps to one unified page.
"""
logical block id of every group maps to one unified page."""
unified_page = worker._test_unified_page
bases = [t.data_ptr() for t in worker._test_tensors]
owned = []
@@ -633,8 +631,7 @@ def test_hetero_ppl_multi_read_writes_stay_within_request_blocks():
(kernel 4, ppl=3) vs remote (P, TP2) logical blocks of 8 tokens (ppl=2),
equal kernel pages, tp_ratio=-2 multi-read with replicated MLA and
TP-sharded KDA state. Every local descriptor of the request's reads must
stay within its own blocks.
"""
stay within its own blocks."""
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
NixlConnectorMetadata,
)
@@ -701,8 +698,7 @@ def _resolve(
"""Resolve a desc id to (region, kind, token_start) where kind is 'attn'
(desc-page sized, sub-block-aligned, in the request's attention blocks)
or 'mamba'. token_start is the request-relative token offset, so local
and remote are comparable even when their kernel blocks differ in size.
"""
and remote are comparable even when their kernel blocks differ in size."""
addr, length, _ = (int(x) for x in desc_arr[int(idx)])
for region, base in enumerate(bases):
off = addr - base
@@ -896,8 +892,7 @@ def _run_hetero_case(
def test_hetero_ppl_token_alignment_sweep(local_block, remote_block, num_tokens):
"""Sweep prompt lengths across block-boundary residues for several
hetero-ppl geometries; assert neighbor-safety, token alignment, and
coverage of every transferred kernel block.
"""
coverage of every transferred kernel block."""
_run_hetero_case(
local_block, kernel=4, remote_block=remote_block, num_tokens=num_tokens
)
@@ -916,8 +911,7 @@ def test_hetero_ppl_with_block_size_ratio(num_tokens):
2). The transfer is clipped at remote sub-block granularity by the
pairing and front-trimmed by _apply_prefix_caching, so the
untransferred tail can span both a partial block and whole blocks —
the case each of the two former zeroing paths handled only half of.
"""
the case each of the two former zeroing paths handled only half of."""
_run_hetero_case(
local_block=24,
kernel=8,
@@ -966,8 +960,7 @@ def test_mla_hybrid_large_ppl_geometry(num_tokens):
"""KimiLinear-scale MLA-hybrid geometry (TP8 prefill -> TP1 decode):
decode (local) logical block 5760 / kernel 64 (ppl=90), prefill
(remote) logical block 768 (ppl=12), tp_ratio=-8 multi-read with
replicated MLA and 8-way TP-sharded KDA state.
"""
replicated MLA and 8-way TP-sharded KDA state."""
_run_hetero_case(
local_block=5760,
kernel=64,
@@ -981,8 +974,7 @@ def test_mla_hybrid_large_ppl_geometry(num_tokens):
def test_mismatched_mla_kernel_page_rejected_for_mla_hybrid():
"""The MLA per-token page is TP-independent, so kernel block lengths
differing by anything other than the block-size ratio must fail the
handshake loudly rather than transfer at mismatched geometry.
"""
handshake loudly rather than transfer at mismatched geometry."""
worker = _make_mla_hybrid_worker(
local_block_size=12, kernel_block_size=4, num_logical_blocks=8
)
@@ -1166,8 +1158,7 @@ def test_csa_linear_registration_discovers_shared_regions():
def test_csa_linear_scratch_descs_follow_the_pages_they_overlay():
"""The scratch ring is addressed by the regions it actually registered in,
not by inferring that it must sit on the MLA pages: overlaying the main KV
pages instead moves its descriptors with it.
"""
pages instead moves its descriptors with it."""
worker = _make_csa_linear_ple_worker(scratch_aliases="main_kv")
# Same MLA regions as before, but the ring now lives in the non-MLA ones.
@@ -1301,8 +1292,7 @@ def test_csa_linear_remote_ple_is_copied_whole():
def _make_ring_worker():
"""Paged MLA group plus a per-layer ring group, no Mamba: the shape of a
model whose sliding-window KV lives in per-request rings.
"""
model whose sliding-window KV lives in per-request rings."""
from unittest.mock import MagicMock
from vllm.config import set_current_vllm_config
@@ -115,8 +115,7 @@ class _BlocksMock:
def _stub_sw_clipping(scheduler) -> None:
"""Make ``get_exchange_clipped_blocks`` a passthrough so tests don't
need the full sliding-window machinery.
"""
need the full sliding-window machinery."""
scheduler.get_exchange_clipped_blocks = lambda block_ids, clip_ssm=True: block_ids
@@ -174,8 +173,7 @@ class TestPushScheduler:
def test_p_side_request_finished_stages_blocks(self):
"""P scheduler pushes blocks into both _finished_request_blocks (lease)
and _newly_finished_push_blocks (metadata for next step).
"""
and _newly_finished_push_blocks (metadata for next step)."""
sched = make_nixl_push_scheduler()
_stub_sw_clipping(sched)
@@ -194,8 +192,7 @@ class TestPushScheduler:
def test_build_connector_meta_drains_both_sides(self):
"""meta.push_registrations and meta.push_finished_blocks are filled
from the staging dicts and the staging dicts are cleared.
"""
from the staging dicts and the staging dicts are cleared."""
sched = make_nixl_push_scheduler()
_stub_sw_clipping(sched)
@@ -290,8 +287,7 @@ class TestPushScheduler:
def test_registration_watchdog_expires(self, caplog):
"""Stale D registrations whose deadline has passed are dropped at
``build_connector_meta`` time.
"""
``build_connector_meta`` time."""
# Watchdog logs a WARNING when it drops the stale entry; that's
# what this test is verifying, so silence it in the test report.
caplog.set_level(
@@ -332,8 +328,7 @@ class TestPushScheduler:
class _StubWriterWorker(NixlPushConnectorWorker):
"""Construct a worker without invoking ``__init__`` so we can drive
the matching/notif logic without bringing up NIXL or torch.
"""
the matching/notif logic without bringing up NIXL or torch."""
@classmethod
def fresh(cls) -> _StubWriterWorker:
@@ -439,8 +434,7 @@ class TestPushWriterRegSend:
"""``_do_send_reg_notif``: the PUSH_REG goes to every handshaked agent of
the P engine, and the agent map is read under ``_handshake_lock`` so the
send can't race the handshake done-callback's insert or a concurrent
stale-engine eviction (X1).
"""
stale-engine eviction (X1)."""
def _reg_send_worker(self) -> tuple[_StubWriterWorker, list]:
w = _StubWriterWorker.fresh()
@@ -485,8 +479,7 @@ class TestPushWriterRegSend:
def test_agent_read_is_serialized_with_handshake_lock(self):
"""While another thread holds ``_handshake_lock`` (as the handshake
done-callback does while writing ``_remote_agents``), the writer's
check-then-send must not run ahead of the critical section.
"""
check-then-send must not run ahead of the critical section."""
w, failures = self._reg_send_worker()
w._remote_agents["prefill-engine"] = {(0, 0): "agent-0"}
rd = _registration_data("req-1")
@@ -596,8 +589,7 @@ class TestPushWriterMatching:
class TestPushWriterStartLoadKv:
def test_finished_blocks_inbox_matches_stashed_registration(self):
"""Run the writer-loop's finished-blocks drain against a
pre-populated _pending_d_registrations entry.
"""
pre-populated _pending_d_registrations entry."""
w = _StubWriterWorker.fresh()
w._pending_d_registrations["req-C"] = _registration_data("req-C")
@@ -622,8 +614,7 @@ class TestPushWriterStartLoadKv:
def test_start_load_kv_enqueues_to_writer(self):
"""``start_load_kv`` should hand registrations + finished blocks
to the writer queues without doing matching itself.
"""
to the writer queues without doing matching itself."""
w = _StubWriterWorker.fresh()
# Stub heartbeats to a no-op; tests don't exercise the heartbeat
# path here.
@@ -681,8 +672,7 @@ def test_do_start_push_kv_defers_then_writes_when_handshake_ready():
NIXL op from the writer or the executor callback); once it resolves the
request is re-queued on ``_deferred_push_inbox`` with the wake set; and on
re-drive with the handshake ready the WRITE is issued with a correct
ReqMeta.
"""
ReqMeta."""
w = _StubWriterWorker.fresh()
w._logical_to_kernel_block_ids = lambda x, ratio: x
xfer_calls: list[dict[str, Any]] = []
@@ -722,8 +712,7 @@ def test_do_start_push_kv_defers_then_writes_when_handshake_ready():
def test_do_start_push_kv_drops_request_on_handshake_failure():
"""Handshake raises: the request is dropped (not re-queued, no WRITE) and
the failure is logged. Blocks are reclaimed by the lease/watchdog, matching
the old blocking behaviour.
"""
the old blocking behaviour."""
w = _StubWriterWorker.fresh()
w._logical_to_kernel_block_ids = lambda x, ratio: x
xfer_calls: list[dict[str, Any]] = []
@@ -745,8 +734,7 @@ def test_do_start_push_kv_drops_request_on_handshake_failure():
def test_writer_loop_drains_deferred_push_inbox():
"""The writer loop drains ``_deferred_push_inbox`` and re-drives
``_do_start_push_kv`` for each entry (event-driven, no polling).
"""
``_do_start_push_kv`` for each entry (event-driven, no polling)."""
w = _StubWriterWorker.fresh()
w.nixl_wrapper = MagicMock()
w.nixl_wrapper.get_new_notifs.return_value = {}
@@ -779,8 +767,7 @@ def test_writer_loop_drains_deferred_push_inbox():
def _eviction_worker(engine_ttl: float) -> NixlPushConnectorWorker:
"""A push worker wired to drive the base ``_ensure_handshake`` eviction
path (``_evict_stale_engines`` + ``_cleanup_remote_engine``).
"""
path (``_evict_stale_engines`` + ``_cleanup_remote_engine``)."""
w = _StubWriterWorker.fresh()
w._engine_ttl = engine_ttl
w._engine_last_active = {}
@@ -802,8 +789,7 @@ def _eviction_worker(engine_ttl: float) -> NixlPushConnectorWorker:
def test_active_push_refreshes_engine_last_active():
"""The base ``_ensure_handshake`` refreshes liveness only on a new
handshake, so an active push to an already-connected engine must refresh
``_engine_last_active`` itself or it is reaped mid-stream (S1).
"""
``_engine_last_active`` itself or it is reaped mid-stream (S1)."""
w = _eviction_worker(engine_ttl=3600.0)
# Already-connected D -> _ensure_handshake returns None, WRITE runs inline.
w._remote_agents["decode-engine"] = {(0, 0): "agent-decode"}
@@ -819,8 +805,7 @@ def test_active_push_refreshes_engine_last_active():
def test_stale_engine_evicted_on_push():
"""A push drives ``_ensure_handshake`` -> ``_evict_stale_engines``, so a D
engine silent past its TTL is reaped -- no _remote_agents / NIXL agent leak
across D scale up/down (S1).
"""
across D scale up/down (S1)."""
w = _eviction_worker(engine_ttl=30.0)
w._remote_agents["D-old"] = {(0, 0): "agent-D-old"}
w._engine_last_active["D-old"] = time.perf_counter() - 10_000.0
@@ -837,8 +822,7 @@ def test_stale_engine_evicted_on_push():
def test_cleanup_remote_engine_pops_under_handshake_lock():
"""Regression (X1): eviction must not remove ``_remote_agents`` entries
while another thread is inside the lock's critical section (e.g. the
push writer reading the map to send a registration).
"""
push writer reading the map to send a registration)."""
w = _eviction_worker(engine_ttl=30.0)
w._remote_agents["D-old"] = {(0, 0): "agent-D-old"}
@@ -863,8 +847,7 @@ def test_cleanup_remote_engine_pops_under_handshake_lock():
class TestPushWriterNotifs:
def test_get_new_notifs_processes_forwarded_completion_notif(self):
"""Non-PUSH_REG notifs forwarded by the writer thread are drained
on the engine main thread inside ``_get_new_notifs``.
"""
on the engine main thread inside ``_get_new_notifs``."""
w = _StubWriterWorker.fresh()
# Pretend the writer thread already forwarded a completion notif
# for a request whose KV is being received.
@@ -910,8 +893,7 @@ class TestPushWriterNotifs:
@staticmethod
def _pollable_worker() -> _StubWriterWorker:
"""Stub worker with the extra base-worker state the real
``get_transfer_results`` needs to poll ``_sending_transfers``.
"""
``get_transfer_results`` needs to poll ``_sending_transfers``."""
w = _StubWriterWorker.fresh()
w.transfer_topo = MagicMock()
w.nixl_wrapper = MagicMock()
@@ -953,8 +935,7 @@ class TestPushWriterNotifs:
@pytest.mark.parametrize(("pp_size", "is_hma"), [(1, False), (2, True)])
def test_completed_push_send_waits_for_consumer_notif(self, pp_size, is_hma):
"""P-side sends are completed by consumer notifs / lease expiry,
not by local WRITE-handle completion.
"""
not by local WRITE-handle completion."""
w = self._pollable_worker()
w.pp_size = pp_size
w._is_hma_required = is_hma
@@ -1025,8 +1006,7 @@ class TestPushSchedulerNegative:
def test_request_finished_unfinished_status_does_not_stage(self):
"""If a request is still RUNNING, request_finished must not stash
blocks for the worker (no push needed).
"""
blocks for the worker (no push needed)."""
sched = make_nixl_push_scheduler()
_stub_sw_clipping(sched)
@@ -1043,8 +1023,7 @@ class TestPushSchedulerNegative:
def test_request_finished_empty_blocks_does_not_arm_lease(self):
"""Empty block-id groups should still complete cleanly without
arming the lease/finished maps.
"""
arming the lease/finished maps."""
sched = make_nixl_push_scheduler()
_stub_sw_clipping(sched)
@@ -1059,8 +1038,7 @@ class TestPushSchedulerNegative:
def test_update_connector_output_unknown_request_is_noop(self):
"""Idempotent cleanup: clearing a request that was never staged
must not raise or mutate other state.
"""
must not raise or mutate other state."""
sched = make_nixl_push_scheduler()
_stub_sw_clipping(sched)
@@ -1093,8 +1071,7 @@ class TestPushWriterNegative:
def test_pop_matching_registration_no_match_when_base_ids_differ(self):
"""A registration whose base id (after stripping the random suffix)
does NOT match the lookup request_id must not be popped.
"""
does NOT match the lookup request_id must not be popped."""
w = _StubWriterWorker.fresh()
# Two unrelated requests: different base UUIDs, so stripping the
# trailing ``-<8 hex>`` suffix still yields different base ids.
@@ -1110,8 +1087,7 @@ class TestPushWriterNegative:
def test_handle_push_reg_with_non_dict_payload_is_dropped(self, caplog):
"""msgpack-encoded non-dict payload (e.g. a list) should be
dropped without raising.
"""
dropped without raising."""
caplog.set_level(
logging.CRITICAL,
logger=("vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_worker"),
@@ -1139,8 +1115,7 @@ class TestPushWriterNegative:
def test_handle_push_reg_idempotent_for_same_request_id(self):
"""Receiving the same PUSH_REG twice (e.g. P retries after a
flake) keeps the entry staged exactly once and never fires.
"""
flake) keeps the entry staged exactly once and never fires."""
w = _StubWriterWorker.fresh()
notif = PUSH_REG_NOTIF_PREFIX + msgspec.msgpack.encode(
_registration_data("req-dup")
@@ -1155,8 +1130,7 @@ class TestPushWriterNegative:
"""``get_transfer_results`` must enqueue every completed request
in ``done_sending`` so the writer can drop stale matching state.
Unlike the happy-path test, this verifies the *cardinality*: N
completed requests -> N evictions, in order.
"""
completed requests -> N evictions, in order."""
w = _StubWriterWorker.fresh()
with patch.object(
NixlPushConnectorWorker.__mro__[1],
@@ -1179,8 +1153,7 @@ class TestPushWriterNegative:
def test_empty_transfer_results_do_not_enqueue_eviction(self):
"""If there's nothing newly done, no eviction should be enqueued.
The wake event IS still set because ``get_finished`` always wakes
the writer to drain notifs.
"""
the writer to drain notifs."""
w = _StubWriterWorker.fresh()
with patch.object(
NixlPushConnectorWorker.__mro__[1],
@@ -1221,8 +1194,7 @@ class TestPushWriterNegative:
def test_get_new_notifs_unknown_request_is_logged_and_skipped(self, caplog):
"""A completion notif for a request the worker doesn't know
about should be logged but not crash.
"""
about should be logged but not crash."""
caplog.set_level(
logging.CRITICAL,
logger=("vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_worker"),
@@ -1254,8 +1226,7 @@ class TestPushWriterNegative:
def test_get_new_notifs_extends_lease_on_heartbeat(self):
"""``HB:`` notifs forwarded by the writer thread must extend the
leases of tracked P-side requests on the engine main thread, and
ignore request IDs that aren't being tracked.
"""
ignore request IDs that aren't being tracked."""
w = _StubWriterWorker.fresh()
w.transfer_topo = MagicMock()
# _handle_heartbeat reads ``self._lease_extension`` (set in the
@@ -1291,13 +1262,11 @@ class TestPushWriterNegative:
class TestPushPipelineParallel:
"""PP-sharded producer: per-stage completion counting, pp_size plumbing,
and remote-region slicing.
"""
and remote-region slicing."""
def test_completion_waits_for_one_notif_per_pp_stage(self):
"""Each PP stage WRITEs its own layers and sends one notif; D must
collect pp_size notifs before reporting the recv done.
"""
collect pp_size notifs before reporting the recv done."""
w = _StubWriterWorker.fresh()
w.transfer_topo = MagicMock()
request_id = "req-pp-2"
@@ -1318,8 +1287,7 @@ class TestPushPipelineParallel:
def test_req_meta_reads_pp_size_from_kv_transfer_params(self):
"""D learns the producer's pp_size from kv_transfer_params (forwarded
by the proxy) and defaults to 1 when absent.
"""
by the proxy) and defaults to 1 when absent."""
metadata = NixlConnectorMetadata()
params = {
"remote_block_ids": ([0],),
@@ -1342,8 +1310,7 @@ class TestPushPipelineParallel:
this worker holds only a contiguous layer slice; add_remote_agent
trims the remote region list to [offset : offset + num_local_regions]
before building descriptors. We stop right after the slice via a
sentinel on the next collaborator call.
"""
sentinel on the next collaborator call."""
block_len = 4096 * 16
w = _StubWriterWorker.fresh() # seeds writer-thread state for teardown
w.pp_size = 2
@@ -1385,8 +1352,7 @@ class TestPushWriterMlaReplication:
"""MLA latent KV is replicated across D's TP ranks, so when D_TP > P_TP
(``tp_ratio < 0``) the producer must *WRITE* the latent into every D rank
it handshook -- not write one and merely notify the rest, which would
leave the un-written ranks decoding against stale KV.
"""
leave the un-written ranks decoding against stale KV."""
@staticmethod
def _mla_worker_writing_to(d_ranks):
@@ -1611,8 +1577,7 @@ class TestPushPrefixCaching:
def test_partial_prefix_hit_end_trims_producer_blocks(self):
"""D registered only its 2 uncomputed suffix blocks; P finished the
full 5-block sequence. P must WRITE its LAST 2 blocks into D's slots.
"""
full 5-block sequence. P must WRITE its LAST 2 blocks into D's slots."""
w, _ = self._worker_driving_xfer()
reg = _registration_data("req-pc", local_block_ids=([500, 501],))
@@ -171,8 +171,7 @@ def test_gpu_memory_nixl_hma(model_name, sw_size):
@pytest.mark.parametrize("model_name", ["google/gemma-3-1b-it"])
def test_gpu_memory_no_nixl_baseline(model_name):
"""Same workload without NixlConnector. Comparing driver-level memory
between this and test_gpu_memory_nixl_hma isolates UCX/NIXL impact.
"""
between this and test_gpu_memory_nixl_hma isolates UCX/NIXL impact."""
from vllm import LLM, SamplingParams
from vllm.distributed.parallel_state import cleanup_dist_env_and_memory
@@ -213,8 +213,7 @@ def _multi_connector_config(swa_enabled: bool = False):
def test_nixl_wins_load_over_cpu_offload():
"""When NixlConnector (index 0) has matched tokens from a remote prefill, it should
win the load: Nixl metadata tracks the recv while CPU offload metadata has no load
scheduled.
"""
scheduled."""
vllm_config, kv_cache_config = _multi_connector_config()
scheduler = create_scheduler(vllm_config, kv_cache_config=kv_cache_config)
mc = scheduler.connector
@@ -251,8 +250,7 @@ def test_nixl_wins_load_over_cpu_offload():
@patch(NIXL_WRAPPER_PATCH, FakeNixlWrapper)
def test_cpu_offload_wins_when_nixl_has_no_match():
"""When NixlConnector returns 0 matched tokens and SimpleCPUOffloadConnector has a
CPU cache hit, the CPU offload connector (index 1) wins the load.
"""
CPU cache hit, the CPU offload connector (index 1) wins the load."""
vllm_config, kv_cache_config = _multi_connector_config()
scheduler = create_scheduler(vllm_config, kv_cache_config=kv_cache_config)
mc = scheduler.connector
@@ -311,8 +309,7 @@ def test_cpu_offload_wins_when_nixl_has_no_match():
def test_request_finished_no_async_save(swa_enabled: bool):
"""A normal request (no P/D) produces no async save from either connector.
MultiConnector returns (False, None) via both request_finished and
request_finished_all_groups, and cleans up _requests_to_connector.
"""
request_finished_all_groups, and cleans up _requests_to_connector."""
from vllm.v1.request import RequestStatus
vllm_config, kv_cache_config = _multi_connector_config(swa_enabled=swa_enabled)
@@ -321,8 +321,7 @@ def test_cpu_offloading(
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA")
def test_cpu_offloading_metrics() -> None:
"""Verify that offloading Prometheus metrics (new flat and deprecated
labeled) are emitted after stores and loads.
"""
labeled) are emitted after stores and loads."""
extra_config: dict = {
"cpu_bytes_to_use": 500 << 20,
"block_size": CPU_BLOCK_SIZES,
@@ -511,8 +510,7 @@ def test_tiering_offloading() -> None:
def test_fs_tiering_offloading(tmp_path) -> None:
"""Tests OffloadingConnector with TieringOffloadingSpec
+ fs secondary tier.
"""
+ fs secondary tier."""
extra_config: dict = {
"cpu_bytes_to_use": 1 << 30,
"block_size": CPU_BLOCK_SIZES,
@@ -704,8 +704,7 @@ def test_async_load_reserves_blocks_for_inflight():
def test_async_loads_both_admitted_when_pool_fits():
"""Sanity: with a pool large enough, the reservation gate admits both async
loads (it is not over-conservative).
"""
loads (it is not over-conservative)."""
vllm_config = create_vllm_config()
BLOCK_SIZE = vllm_config.cache_config.block_size
scheduler = create_scheduler(vllm_config, num_blocks=64)
@@ -129,8 +129,7 @@ def _make_ring_kv_cache_config(num_blocks: int = 16) -> KVCacheConfig:
def test_ring_scratch_group_is_accepted_and_never_offloaded() -> None:
"""The ring group's capacity is not a token granularity, it holds no
hashed blocks, and it never participates in CPU hits.
"""
hashed blocks, and it never participates in CPU hits."""
vllm_config = create_vllm_config(
kv_connector="SimpleCPUOffloadConnector",
kv_role="kv_both",
@@ -235,8 +235,7 @@ class TestMambaPlanSplitHandles:
def test_hetero_block_size_splits(self):
"""With a block-size ratio, single-source FA sub-block descs pass
through whole; SSM descs are unexpanded and split per source.
"""
through whole; SSM descs are unexpanded and split per source."""
plan = TPMapping(
source_ranks_per_group=((0,), (0, 1)),
all_source_ranks=(0, 1),
@@ -271,8 +270,7 @@ class TestMambaPlanSplitHandles:
def test_hetero_block_size_head_sharded_asserts(self):
"""Head-sharded FA reads (multiple FA sources) are incompatible with
a block-size mismatch and must fail loudly.
"""
a block-size mismatch and must fail loudly."""
plan = TPMapping(
source_ranks_per_group=((0, 1), (0, 1)),
all_source_ranks=(0, 1),
@@ -15,8 +15,7 @@ from vllm.v1.kv_offload.cpu.policies.lru import LRUCachePolicy
class _DummyCachePolicy(CachePolicy):
"""Minimal CachePolicy for CachePolicyFactory registration tests. Loaded
by module path, so it must be importable at module scope (mirrors
tests/v1/kv_offload/test_factory.py's SingleArgExternalOffloadingSpec).
"""
tests/v1/kv_offload/test_factory.py's SingleArgExternalOffloadingSpec)."""
def __init__(self, cache_capacity: int) -> None:
self.cache_capacity = cache_capacity
@@ -55,8 +54,7 @@ class TestCachePolicyFactory:
def test_pre_registered_policies_can_be_imported(self):
"""If someone moves a policy module but forgets to update
factory.py, CI fails.
"""
factory.py, CI fails."""
for name in CachePolicyFactory._registry:
cls = CachePolicyFactory._registry[name]()
assert issubclass(cls, CachePolicy)
@@ -90,8 +88,7 @@ class TestCachePolicyFactory:
register_cache_policy() call -- this is how external projects
integrate a custom CachePolicy without forking/patching vLLM.
Mirrors tests/v1/kv_offload/test_factory.py's
test_dynamic_load_via_spec_module_path.
"""
test_dynamic_load_via_spec_module_path."""
policy_cls = CachePolicyFactory.get_cache_policy_cls(
"_DummyCachePolicy", "tests.v1.kv_offload.cpu.policies.test_factory"
)
@@ -99,8 +96,7 @@ class TestCachePolicyFactory:
def test_manager_resolves_policy_via_module_path(self):
"""End-to-end: CPUOffloadingManager resolves an unregistered policy
purely from cache_policy_module_path.
"""
purely from cache_policy_module_path."""
manager = CPUOffloadingManager(
num_chunks=4,
cache_policy="_DummyCachePolicy",
@@ -110,7 +106,6 @@ class TestCachePolicyFactory:
def test_unregistered_policy_without_module_path_raises(self):
"""eviction_policy not in registry + no cache_policy_module_path ->
ValueError, same shape as the OffloadingSpecFactory error path.
"""
ValueError, same shape as the OffloadingSpecFactory error path."""
with pytest.raises(ValueError, match="Unknown cache policy"):
CachePolicyFactory.get_cache_policy_cls("nonexistent", None)
@@ -126,8 +126,7 @@ def _canonical_handler(gpu_tensor, cpu_tensor, mapping, gpu_to_cpu):
def test_gpu_roundtrip_assembles_canonical_page_across_ranks():
"""Two TP2-style rank handlers must scatter into one shared canonical
CPU page such that each rank reloads bit-exact and a whole-page (TP1)
reader sees both shards — the cross-topology contract.
"""
reader sees both shards — the cross-topology contract."""
torch.manual_seed(0)
num_blocks = 4
gpu_rank = [
@@ -204,8 +203,7 @@ def _nhd_shard_mapping(tp: int, rank: int) -> CanonicalPageMapping:
def _head_shard(full_kv: torch.Tensor, tp: int, rank: int) -> torch.Tensor:
"""This rank's local page rows out of the (blocks, 2, tokens, heads,
head_bytes) ground truth.
"""
head_bytes) ground truth."""
local_heads = _TOTAL_HEADS // tp
shard = full_kv[:, :, :, rank * local_heads : (rank + 1) * local_heads, :]
return shard.reshape(full_kv.shape[0], -1).contiguous()
@@ -216,8 +214,7 @@ def _head_shard(full_kv: torch.Tensor, tp: int, rank: int) -> torch.Tensor:
def test_cross_topology_roundtrip(writer_tp: int, reader_tp: int):
"""KV written at one tp must be readable at another: writer ranks scatter
head shards into a shared canonical region, reader ranks gather their own
shards, and every reader must see the writers' ground-truth bytes.
"""
shards, and every reader must see the writers' ground-truth bytes."""
torch.manual_seed(0)
num_blocks = 3
full_kv = torch.randint(
@@ -167,8 +167,7 @@ def _mp_race_construct_and_write(
) -> None:
"""Race to construct a SharedOffloadRegion, write fill_value, then wait
for the parent's cleanup signal before tearing down. The wait gives the
parent a window to read the raw mmap before the creator removes the file.
"""
parent a window to read the raw mmap before the creator removes the file."""
try:
region = SharedOffloadRegion(
engine_id=engine_id,
@@ -197,8 +196,7 @@ def _mp_barrier_construct_and_hold(
replicated: bool = False,
) -> None:
"""Construct with a real cross-process barrier, write, then hold the
mapping (no cleanup) until the parent SIGKILLs this process.
"""
mapping (no cleanup) until the parent SIGKILLs this process."""
try:
populate_calls = 0
get_populate_write_fn = region_module._get_populate_write_fn
@@ -252,8 +250,7 @@ def iid():
def test_create_next_worker_view_shape_and_stride(iid):
"""Returned tensor must have shape (num_chunks, tensor_page_size) and
stride (row_stride, 1) where row_stride = cpu_page_size * num_workers.
"""
stride (row_stride, 1) where row_stride = cpu_page_size * num_workers."""
with _region(iid, num_chunks=4, cpu_page_size=2 * PAGE_SIZE) as r:
t = r.create_next_worker_view(PAGE_SIZE)
assert t.shape == (4, PAGE_SIZE)
@@ -293,8 +290,7 @@ def test_create_next_worker_view_row_stride_with_multiple_workers(iid):
def test_create_next_worker_view_cursor_advances(iid):
"""Each create_next_worker_view call must advance _worker_offset by
tensor_page_size.
"""
tensor_page_size."""
with _region(iid, cpu_page_size=3 * PAGE_SIZE) as r:
assert r._worker_offset == 0
r.create_next_worker_view(PAGE_SIZE)
@@ -351,8 +347,7 @@ def test_create_next_worker_view_overflow_does_not_mutate_cursor(iid):
def test_create_next_worker_view_write_visible_in_raw_mmap(iid):
"""Writes into a create_next_worker_view view must appear at the correct
raw mmap offset
"""
raw mmap offset"""
with _region(iid, num_chunks=4) as r:
t = r.create_next_worker_view(PAGE_SIZE)
t[2, :] = 42 # write to chunk row 2
@@ -385,8 +380,7 @@ def test_create_next_worker_view_multi_tensor_layout(iid):
def test_create_next_worker_view_multiprocess_slots(iid):
"""Each worker process calls create_next_worker_view and writes distinct data;
the parent verifies each slot lands at the correct interleaved offset.
"""
the parent verifies each slot lands at the correct interleaved offset."""
num_workers = 2
num_chunks = 4
@@ -671,8 +665,7 @@ def test_multi_worker_race_shared_memory_visible(iid):
def test_multiprocess_race_construct_and_write(iid):
"""N processes race to construct the same SharedOffloadRegion, each writes
fill_value = rank+1 into their slot; parent verifies interleaved layout.
"""
fill_value = rank+1 into their slot; parent verifies interleaved layout."""
num_workers = 4
num_chunks = 3
@@ -798,8 +791,7 @@ def test_cleanup_unregisters_every_pinned_chunk(iid, monkeypatch):
def test_cleanup_after_create_next_worker_view_releases_mmap(iid):
"""cleanup() must close the mmap even after create_next_worker_view was called.
create_next_worker_view returns a view that shares storage with _base; both must be
released before mmap.close() can succeed.
"""
released before mmap.close() can succeed."""
r = _make_region(iid)
mmap_obj = r.mmap_obj
@@ -978,8 +970,7 @@ def test_ftruncate_failure_cleans_up_creator(monkeypatch):
def test_backing_file_unlinked_after_barrier(iid):
"""The file must exist until the barrier releases (late joiners need the
name) and be gone right after, so no exit path can leak it.
"""
name) and be gone right after, so no exit path can leak it."""
path = f"/dev/shm/vllm_offload_{iid}.mmap"
seen_at_barrier = []
@@ -1001,8 +992,7 @@ def test_backing_file_unlinked_after_barrier(iid):
def test_barrier_failure_unlinks_creator_and_raises(iid):
"""A failed rendezvous must remove the creator's file and re-raise, not
leave a stub that wedges the next start in _wait_for_file_size.
"""
leave a stub that wedges the next start in _wait_for_file_size."""
path = f"/dev/shm/vllm_offload_{iid}.mmap"
with pytest.raises(RuntimeError, match="peer died"):
_make_region(iid, barrier=MagicMock(side_effect=RuntimeError("peer died")))
@@ -1012,8 +1002,7 @@ def test_barrier_failure_unlinks_creator_and_raises(iid):
@pytest.mark.parametrize("replicated", [False, True])
def test_mp_barrier_unlinks_file_and_survives_sigkill(iid, replicated):
"""With a real cross-process barrier every worker maps one shared inode,
the name is gone while they run, and SIGKILL leaks nothing.
"""
the name is gone while they run, and SIGKILL leaks nothing."""
num_workers = 2
path = f"/dev/shm/vllm_offload_{iid}.mmap"
ctx = get_mp_context()
@@ -1052,8 +1041,7 @@ def test_mp_barrier_unlinks_file_and_survives_sigkill(iid, replicated):
@pytest.mark.parametrize("failure_phase", ["shm", "memory", "population"])
def test_setup_failure_before_barrier_releases_peers(iid, monkeypatch, failure_phase):
"""A worker that dies before the rendezvous must still arrive at the
barrier, or its peers block in the collective until it times out.
"""
barrier, or its peers block in the collective until it times out."""
failure = MagicMock(side_effect=ValueError("Insufficient space"))
memory_check = failure if failure_phase == "memory" else None
if failure_phase == "shm":
@@ -1084,8 +1072,7 @@ def test_setup_failure_before_barrier_releases_peers(iid, monkeypatch, failure_p
def test_mmap_failure_unlinks_creator_before_releasing_peers(iid, monkeypatch):
"""A creator that fails after sizing the file must drop it before arriving
at the barrier, so the next start does not land on a stale file.
"""
at the barrier, so the next start does not land on a stale file."""
path = f"/dev/shm/vllm_offload_{iid}.mmap"
monkeypatch.setattr("mmap.mmap", MagicMock(side_effect=OSError("mmap")))
seen_at_barrier = []
@@ -1098,8 +1085,7 @@ def test_mmap_failure_unlinks_creator_before_releasing_peers(iid, monkeypatch):
def test_barrier_release_failure_keeps_original_error(iid, monkeypatch):
"""When releasing the peers fails too, the setup error that explains the
failure must be the one that propagates.
"""
failure must be the one that propagates."""
import vllm.v1.kv_offload.cpu.shared_offload_region as region
monkeypatch.setattr(
+31 -62
View File
@@ -48,8 +48,7 @@ def _remote_prefiller_kv_params(
kv_request_id: str = "req-1",
) -> dict:
"""Decoder-side kv_transfer_params: ``remote_prefiller`` sub-dict carries
kv_request_id + remote_host + remote_port.
"""
kv_request_id + remote_host + remote_port."""
return {
"remote_prefiller": {
"kv_request_id": kv_request_id,
@@ -65,8 +64,7 @@ def _remote_kv_source_kv_params(
kv_request_id: str = "req-1",
) -> dict:
"""Symmetric-P2P consumer kv_transfer_params: ``remote_kv_source`` sub-dict has
the same shape as ``remote_prefiller`` (kv_request_id + remote_host + port).
"""
the same shape as ``remote_prefiller`` (kv_request_id + remote_host + port)."""
return {
"remote_kv_source": {
"kv_request_id": kv_request_id,
@@ -78,8 +76,7 @@ def _remote_kv_source_kv_params(
def _remote_decoder_kv_params(kv_request_id: str = "req-1") -> dict:
"""Prefiller-side kv_transfer_params: ``remote_decoder`` sub-dict carries
kv_request_id only.
"""
kv_request_id only."""
return {"remote_decoder": {"kv_request_id": kv_request_id}}
@@ -244,8 +241,7 @@ class TestLookup:
def test_lookup_returns_miss_without_prefill_key(self):
"""No ``remote_prefiller`` sub-dict means the request was not routed for
remote prefill — local prefill should run instead, so lookup()
returns MISS even when a stale ``remote_decoder`` block is present.
"""
returns MISS even when a stale ``remote_decoder`` block is present."""
mgr = _make_manager()
ctx = _req_context(kv_params=_remote_decoder_kv_params())
assert mgr.lookup(b"key", ctx) is LookupResult.MISS
@@ -313,8 +309,7 @@ class TestServeExternalRequests:
def test_flushes_failed_serve_ctxs_then_serves_each_session(self):
"""serve_external_requests releases the failed serves left by
reaped sessions via parent.on_request_finished (clearing the
queue), then delegates to every live session with the same parent.
"""
queue), then delegates to every live session with the same parent."""
mgr = _make_manager()
ctx = ReqContext(req_id="p2p:peer:req-1:lu1")
mgr._failed_serve_ctxs = [ctx]
@@ -368,8 +363,7 @@ class TestSubmitStore:
def test_no_binding_yet_parks_in_unbound_stores(self):
"""submit_store without a bound session buffers the batch keyed
by kv_request_id; no session is pre-created (peer_id is unknown
to the producer at store time).
"""
to the producer at store time)."""
mgr = _make_manager()
job = _job_metadata(
job_id=1,
@@ -396,8 +390,7 @@ class TestSubmitStore:
def test_routes_to_bound_session(self):
"""If a session has already received FetchMsg for this
kv_request_id (so _kv_to_session is populated), submit_store
forwards directly to that session rather than re-buffering.
"""
forwards directly to that session rather than re-buffering."""
mgr = _make_manager()
bound = _FakeSession(peer_id="10.0.0.1:8000", connected=True)
mgr._kv_to_session["req-1"] = bound # type: ignore[assignment]
@@ -420,8 +413,7 @@ class TestSubmitStore:
def test_extra_top_level_keys_are_ignored(self):
"""Producer-side kv_transfer_params should not pre-create a
session even when a stale caller still passes a top-level
``remote_host``/``remote_port`` next to ``remote_decoder``.
"""
``remote_host``/``remote_port`` next to ``remote_decoder``."""
mgr = _make_manager()
params = _remote_decoder_kv_params()
params["remote_host"] = "stale"
@@ -465,8 +457,7 @@ class TestSubmitLoad:
def test_happy_path_with_active_session(self):
"""When the peer's session exists, submit_load forwards to
session.request_blocks and does NOT add a finished result yet.
"""
session.request_blocks and does NOT add a finished result yet."""
mgr = _make_manager()
peer_id = "10.0.0.1:8000"
existing = _FakeSession(peer_id=peer_id, connected=True)
@@ -486,8 +477,7 @@ class TestSubmitLoad:
def test_missing_consumer_flag_fails(self):
"""Peer fields present but neither do_remote_prefill nor
do_p2p_fetch is set — submit_load fails the job rather than
emit a stray FetchMsg.
"""
emit a stray FetchMsg."""
mgr = _make_manager()
params = {
"remote_host": "10.0.0.1",
@@ -531,8 +521,7 @@ class TestOnRequestFinished:
def test_decoder_side_calls_session_finish_request(self):
"""Decoder-side finish (``remote_prefiller`` set) still routes via peer_id
because the consumer addresses the producer it loaded from. The
session's finish_request cancels the client-role load.
"""
session's finish_request cancels the client-role load."""
mgr = _make_manager()
peer_id = "10.0.0.1:8000"
session = _FakeSession(peer_id=peer_id)
@@ -544,8 +533,7 @@ class TestOnRequestFinished:
def test_p2p_consumer_side_calls_session_finish_request(self):
"""Symmetric-P2P consumer finish (``remote_kv_source`` set) routes via peer_id
so the session drops any pending lookups (cancel_lookups) and
cancels any inbound load.
"""
cancels any inbound load."""
mgr = _make_manager()
peer_id = "10.0.0.1:8000"
session = _FakeSession(peer_id=peer_id)
@@ -556,8 +544,7 @@ class TestOnRequestFinished:
def test_prefiller_bound_id_routes_via_kv_to_session(self):
"""Prefiller-side finish for an id whose session is already bound
(FetchMsg received) routes via _kv_to_session and pops the entry.
"""
(FetchMsg received) routes via _kv_to_session and pops the entry."""
mgr = _make_manager()
bound = _FakeSession(peer_id="some-peer:1", connected=True)
mgr._kv_to_session["req-1"] = bound # type: ignore[assignment]
@@ -571,8 +558,7 @@ class TestOnRequestFinished:
and no session binding is a no-op on `_unbound_stores`. The
parked batches survive until a peer fetches them or the
`_reap_unbound_stores` timeout fires — `on_request_finished`
must not evict them.
"""
must not evict them."""
from vllm.v1.kv_offload.tiering.p2p.manager import _UnboundStoreBatch
mgr = _make_manager()
@@ -742,8 +728,7 @@ class TestGetFinished:
def test_reap_fails_probes(self):
"""A reaped session's in-flight lookups land in _failed_req_ids so
the consumer's lookup() returns MISS instead of RETRY forever.
"""
the consumer's lookup() returns MISS instead of RETRY forever."""
class FakeData:
def remove_remote_peer(self, pid):
@@ -778,8 +763,7 @@ class TestGetFinished:
def test_unbound_store_reaped_after_timeout(self):
"""Unbound stores past the reap deadline surface as failed and their
kv_request_id lands in _reaped_stores so a late FetchMsg is rejected
instead of parking demand nothing can satisfy.
"""
instead of parking demand nothing can satisfy."""
from vllm.v1.kv_offload.tiering.p2p.manager import _UnboundStoreBatch
mgr = self._make()
@@ -801,8 +785,7 @@ class TestGetFinished:
def test_submit_store_parks_unbound_batch(self):
"""submit_store on an unbound id appends a batch with a fresh
submitted_at stamp so the unbound-store sweep can age it out.
"""
submitted_at stamp so the unbound-store sweep can age it out."""
mgr = _make_manager()
job = _job_metadata(
job_id=1, kv_params=_remote_decoder_kv_params(kv_request_id="req-1")
@@ -824,21 +807,18 @@ class TestHasPendingWork:
"""has_pending_work() must always return True so the engine keeps
ticking the offload pipeline — that's the only thread driving
_control.poll() (incoming peer connects) and session.poll()
(incoming fetch messages on existing sessions).
"""
(incoming fetch messages on existing sessions)."""
def test_returns_true_unconditionally_to_keep_engine_ticking(self):
"""Even with no sessions and no jobs, has_pending_work() returns
True so the engine keeps calling get_finished_jobs(), which is
what drives _control.poll() for inbound peer connects.
"""
what drives _control.poll() for inbound peer connects."""
mgr = _make_manager()
assert mgr.has_pending_work() is True
def test_returns_true_even_when_sessions_present(self):
"""The result is the same regardless of session state — there is
no 'idle' branch.
"""
no 'idle' branch."""
mgr = _make_manager()
mgr._sessions["peer:1"] = _FakeSession(peer_id="peer:1") # type: ignore[assignment]
assert mgr.has_pending_work() is True
@@ -851,8 +831,7 @@ class TestHasPendingWork:
class _ShutdownFakeData:
"""Fake DataTransport that records cancel/poll/close calls and
drives the wait-cancel loop with a scriptable `still` queue.
"""
drives the wait-cancel loop with a scriptable `still` queue."""
def __init__(self, still_queue: list[list[int]] | None = None) -> None:
# Each list in still_queue is the set of ids the next
@@ -896,8 +875,7 @@ class _ShutdownFakeControl:
class TestShutdownDrain:
"""shutdown() drains inflight transfers via cancel(mode='wait')
before calling _data.close(), with a 3s deadline fallback to
cancel(mode='immediate').
"""
cancel(mode='immediate')."""
def _prep(
self,
@@ -1316,8 +1294,7 @@ class _RecordingConn:
class TestAcceptNewPeers:
"""A second inbound from an already-connected peer is rejected and the
new conn is closed; the existing session is left untouched.
"""
new conn is closed; the existing session is left untouched."""
def test_duplicate_connection_is_closed_and_existing_session_untouched(self):
mgr = _make_manager()
@@ -1339,8 +1316,7 @@ class TestAcceptNewPeers:
"""An inbound conn from a peer with no existing session creates
a fresh connected session and registers it under conn.peer_id.
The prefiller has no pre-created pending session anymore — the
first signal of a peer's existence is its inbound connection.
"""
first signal of a peer's existence is its inbound connection."""
class FakeData:
block_len = 4096
@@ -1387,8 +1363,7 @@ class TestAcceptNewPeers:
class TestPollOnce:
"""_poll_once must drain control, accept new peers, poll every session,
surface results, and reap dead sessions — in that order.
"""
surface results, and reap dead sessions — in that order."""
def test_orchestrates_accept_poll_and_reap(self):
mgr = _make_manager()
@@ -1444,8 +1419,7 @@ class TestPollOnce:
def test_new_fetch_id_binds_and_replays_unbound_batches(self):
"""When session.poll() reports a kv_request_id whose FetchMsg
arrived this tick, the manager binds it to that session and
replays every parked submit_store batch via add_stored_blocks.
"""
replays every parked submit_store batch via add_stored_blocks."""
from vllm.v1.kv_offload.tiering.p2p.manager import _UnboundStoreBatch
mgr = _make_manager()
@@ -1479,8 +1453,7 @@ class TestPollOnce:
def test_new_fetch_id_with_no_unbound_still_binds(self):
"""A FetchMsg for a kv_request_id with no parked batches still
records the binding so subsequent submit_stores route fast.
"""
records the binding so subsequent submit_stores route fast."""
mgr = _make_manager()
peer = "10.0.0.1:8000"
sess = _FakeSession(
@@ -1503,8 +1476,7 @@ class TestPollOnce:
def test_failed_load_records_kv_request_id(self):
"""A LoadResult(success=False) from session.poll() must add its
kv_request_id to _failed_req_ids so future lookups return MISS.
"""
kv_request_id to _failed_req_ids so future lookups return MISS."""
mgr = _make_manager()
peer = "10.0.0.1:8000"
sess = _FakeSession(
@@ -1568,8 +1540,7 @@ class TestDrainJobs:
def test_logs_warning_after_5s_then_completes(self, monkeypatch):
"""A session that stays inflight past 5s triggers the warning, and
once it clears the loop returns.
"""
once it clears the loop returns."""
mgr = _make_manager()
mgr._control = _DrainCtrl() # type: ignore[assignment]
sess = _FakeSession(peer_id="peer:1")
@@ -1627,8 +1598,7 @@ class TestDrainJobs:
class TestOnScheduleEnd:
def test_is_noop(self):
"""on_schedule_end is a documented no-op; just confirm it doesn't
raise and doesn't mutate state.
"""
raise and doesn't mutate state."""
mgr = _make_manager()
before_sessions = dict(mgr._sessions)
before_jobs = list(mgr._finished_jobs)
@@ -1654,8 +1624,7 @@ class TestConnectionDeathMidTransfer:
prefiller-side store no longer travels through the session at store
time (it's parked in _unbound_stores keyed by kv_request_id), so its
cleanup on connection death is via on_request_finished or the
unbound-store timeout — covered separately below.
"""
unbound-store timeout — covered separately below."""
def test_dead_connection_with_pending_work_surfaces_failures(self):
mgr_a, mgr_b = _build_paired_managers()
@@ -626,8 +626,7 @@ class TestClientFlows:
"""collect_results / has_active_loads use the _active_loads work-list,
armed when a fetch is issued and discarded exactly when its load
clears — a probe-only request never enters it, and completion empties
it while the entry may briefly linger for GC.
"""
it while the entry may briefly linger for GC."""
session, conn, _ = _make_session()
_activate(session, conn)
client = session._client
@@ -704,8 +703,7 @@ class TestClientFlows:
def test_load_abort_ack_clears_request(self):
"""After load timeout sends AbortFetch, an arriving AbortAckMsg from
the peer surfaces the failure cleanly and removes the request from
_requests — covers the on_abort_ack arrival path.
"""
_requests — covers the on_abort_ack arrival path."""
session, conn, _ = _make_session()
_activate(session, conn)
session.request_blocks(
@@ -742,8 +740,7 @@ class TestLookupFlow:
def test_aggregate_flush_resolve_round_trip(self):
"""register_lookup → flush sends one LookupMsg → response
resolves entries → register_lookup returns the cached bool on
every call, and repeat probes never re-issue a LookupMsg.
"""
every call, and repeat probes never re-issue a LookupMsg."""
session, conn, _ = _make_session()
_activate(session, conn)
@@ -796,8 +793,7 @@ class TestLookupFlow:
def test_request_blocks_clears_probe_cache(self):
"""A resolved HIT probe is popped when its fetch is issued, so a
re-scheduled request re-probes instead of trusting the stale True
(the served block is unpinned and may have been evicted).
"""
(the served block is unpinned and may have been evicted)."""
session, conn, _ = _make_session()
_activate(session, conn)
@@ -883,8 +879,7 @@ class TestLookupFlow:
def test_multiple_lookup_msgs_across_steps(self):
"""A request's block set may be discovered across scheduler steps:
each step that registers new keys flushes its own LookupMsg for
the same kv_request_id, carrying only the newly-probed keys.
"""
the same kv_request_id, carrying only the newly-probed keys."""
session, conn, _ = _make_session()
_activate(session, conn)
@@ -912,8 +907,7 @@ class TestLookupFlow:
def test_split_response_resolves_across_messages(self):
"""Producer may answer one LookupMsg's keys across multiple
LookupRespMsgs — pairs are self-describing so each lands.
"""
LookupRespMsgs — pairs are self-describing so each lands."""
session, conn, _ = _make_session()
_activate(session, conn)
@@ -960,8 +954,7 @@ class TestLookupFlow:
def test_finish_after_flushed_lookup_sends_empty_fetch(self):
"""LookupMsg flushed but no FetchMsg sent (all-miss case) →
finish_request emits an empty FetchMsg so the peer can drop its
lookup state and call parent.on_request_finished.
"""
lookup state and call parent.on_request_finished."""
session, conn, _ = _make_session()
_activate(session, conn)
@@ -979,8 +972,7 @@ class TestLookupFlow:
def test_finish_without_flushed_lookup_sends_no_fetch(self):
"""No LookupMsg was ever sent → finish_request must not emit an
empty FetchMsg (the peer has no state to release).
"""
empty FetchMsg (the peer has no state to release)."""
session, conn, _ = _make_session()
_activate(session, conn)
@@ -995,8 +987,7 @@ class TestLookupFlow:
def test_finish_after_real_fetch_sends_no_second_fetch(self):
"""A real FetchMsg was already sent for the id → finish_request
must not emit a second (empty) FetchMsg.
"""
must not emit a second (empty) FetchMsg."""
session, conn, _ = _make_session()
_activate(session, conn)
@@ -1027,8 +1018,7 @@ class TestLookupFlow:
"""``poll()`` only enqueues an inbound LookupMsg — no response is
sent until ``serve_external_requests``. With an all-miss parent
the aggregated LookupRespMsg carries the same keys and
``hits=[False, ...]``.
"""
``hits=[False, ...]``."""
session, conn, _ = _make_session()
_activate(session, conn)
@@ -1088,8 +1078,7 @@ class TestServerLookupHandling:
def test_immediate_hits_create_one_store_job(self):
"""All-HIT batch: one create_store_job call with all keys, one
LookupRespMsg with hits=[True]*N, on_request_finished fires at the
end of serve, and `available` is populated for the eventual fetch.
"""
end of serve, and `available` is populated for the eventual fetch."""
cb = FakeParent(stored={b"hA": 1, b"hB": 2, b"hC": 3})
session, conn, _ = _make_session()
_activate(session, conn)
@@ -1119,8 +1108,7 @@ class TestServerLookupHandling:
def test_all_misses_no_store_job_finish_fires(self):
"""All-MISS batch: no create_store_job call; one LookupRespMsg
with hits=[False]*N; on_request_finished fires at end of serve.
"""
with hits=[False]*N; on_request_finished fires at end of serve."""
cb = FakeParent()
session, conn, _ = _make_session()
_activate(session, conn)
@@ -1140,8 +1128,7 @@ class TestServerLookupHandling:
"""HIT/MISS resolutions do not go out on first sight when any
key is still HIT_PENDING / RETRY. The lookup parks until every
key has settled (or the deadline fires), then one
LookupRespMsg carries all keys in wire order.
"""
LookupRespMsg carries all keys in wire order."""
cb = FakeParent(
stored={b"hA": 1},
pending={b"hB"},
@@ -1170,8 +1157,7 @@ class TestServerLookupHandling:
the deferred aggregate response: one LookupRespMsg carrying
both keys in wire order, and one create_store_job call per
HIT (the second HIT is pinned when it resolves, not when the
response goes out).
"""
response goes out)."""
cb = FakeParent(stored={b"hA": 1}, pending={b"hB"})
session, conn, _ = _make_session()
_activate(session, conn)
@@ -1207,8 +1193,7 @@ class TestServerLookupHandling:
def test_pending_timeout_replies_miss_no_store_job(self):
"""A HIT_PENDING key that stays pending past the batch
``deadline`` is force-MISS and never pinned; the deferred
aggregate response fires with hits=[False].
"""
aggregate response fires with hits=[False]."""
cb = FakeParent(pending={b"hA"})
session, conn, _ = _make_session()
_activate(session, conn)
@@ -1235,8 +1220,7 @@ class TestServerLookupHandling:
def test_finish_request_called_per_lookup_msg_not_per_kv_request_id(self):
"""Two LookupMsgs for the same kv_request_id get distinct ctxs
and two on_request_finished calls (one per batch).
"""
and two on_request_finished calls (one per batch)."""
cb = FakeParent(stored={b"hA": 1, b"hB": 2})
session, conn, _ = _make_session()
_activate(session, conn)
@@ -1260,8 +1244,7 @@ class TestServerLookupHandling:
"""Tearing the session down with a parked batch returns the
synthetic ctx as a failed serve (no parent handle at teardown) so
the manager can release the TieringManager's state on its next
serve.
"""
serve."""
cb = FakeParent(pending={b"hA"})
session, conn, _ = _make_session()
_activate(session, conn)
@@ -1282,8 +1265,7 @@ class TestServerLookupHandling:
def test_wire_finish_drops_pending_batches_for_kv_request_id(self):
"""``ServerRole.finish(kv_request_id)`` drops every parked batch
whose kv_request_id matches and queues its ctx for the next
serve's on_request_finished.
"""
serve's on_request_finished."""
cb = FakeParent(pending={b"hA", b"hB"})
session, conn, _ = _make_session()
_activate(session, conn)
@@ -1314,8 +1296,7 @@ class TestServerLookupHandling:
def test_incoming_fetch_drops_pending_lookups_for_kv_request_id(self):
"""A peer FetchMsg terminates the lookup phase for its id: parked
lookups with matching kv_request_id are dropped and their
ctx queued for on_request_finished; other kv_request_ids untouched.
"""
ctx queued for on_request_finished; other kv_request_ids untouched."""
cb = FakeParent(pending={b"hA", b"hB"})
session, conn, _ = _make_session()
_activate(session, conn)
@@ -1357,8 +1338,7 @@ class TestServerLookupHandling:
def test_lookup_then_fetch_round_trip_emits_store_result(self):
"""End-to-end: lookup pins primary slots → fetch matches them →
NIXL transfer completes → StoreResult surfaces with the
create_store_job's job_id (the engine releases the pin).
"""
create_store_job's job_id (the engine releases the pin)."""
cb = FakeParent(stored={b"hA": 7, b"hB": 8})
session, conn, transport = _make_session()
_activate(session, conn)
@@ -1480,8 +1460,7 @@ class TestServerFlows:
def test_abort_fetch_defers_ack_when_cancel_pending(self):
"""If cancel(mode='wait') reports still-inflight tids, the ack is
deferred and the abort is parked (abort_started_at set).
"""
deferred and the abort is parked (abort_started_at set)."""
session, conn, transport = _make_session()
_activate(session, conn)
# Seed an inflight transfer for req-1 that the transport pretends
@@ -1516,8 +1495,7 @@ class TestServerFlows:
def test_abort_fetch_acks_after_drain(self):
"""Once the transport reports the tid as DONE the parked abort
completes and AbortAckMsg is sent.
"""
completes and AbortAckMsg is sent."""
session, conn, transport = _make_session()
_activate(session, conn)
tid = 42
@@ -1556,8 +1534,7 @@ class TestServerFlows:
def test_abort_fetch_force_cancels_after_timeout(self):
"""If wait-mode never drains, the deadline forces immediate
cancel and an ack is still sent.
"""
cancel and an ack is still sent."""
session, conn, transport = _make_session()
_activate(session, conn)
tid = 42
@@ -1599,8 +1576,7 @@ class TestServerFlows:
def test_abort_fetch_idempotent_while_draining(self):
"""Receiving AbortFetchMsg twice for the same kv_request_id
keeps a single pending entry and produces a single ack.
"""
keeps a single pending entry and produces a single ack."""
session, conn, transport = _make_session()
_activate(session, conn)
tid = 42
@@ -1658,8 +1634,7 @@ class TestServerFlows:
def test_store_timeout_then_late_completion_no_duplicate(self):
"""A job timed out by _timeout_pending_store_jobs must not also
emit a contradictory StoreResult(success=True) when the transport
later reports the same transfer as done.
"""
later reports the same transfer as done."""
session, conn, transport = _make_session()
_activate(session, conn)
session.add_stored_blocks("req-1", [b"k1"], [0], job_id=1)
@@ -1692,8 +1667,7 @@ class TestServerFlows:
def test_store_timeout_then_late_failure_no_duplicate(self):
"""Symmetric guard: a timed-out job must not also emit a second
StoreResult(success=False) when the transport later reports the
same transfer as failed.
"""
same transfer as failed."""
session, conn, transport = _make_session()
_activate(session, conn)
session.add_stored_blocks("req-1", [b"k1"], [0], job_id=1)
@@ -1736,8 +1710,7 @@ class TestFinishRequestServerSide:
def test_no_inflight_unmatched_demand_sends_failure(self):
"""finish_request with unmatched demand and no inflight ->
immediate TransferDoneMsg(success=False); _outbound cleared.
"""
immediate TransferDoneMsg(success=False); _outbound cleared."""
session, conn, _ = _make_session()
_activate(session, conn)
# Decoder demanded a block we never stored.
@@ -1763,8 +1736,7 @@ class TestFinishRequestServerSide:
def test_with_inflight_defers_then_fires_on_last_transfer(self):
"""finish_request with inflight defers; last transfer fires the
early-fail message and clears _outbound.
"""
early-fail message and clears _outbound."""
session, conn, transport = _make_session()
_activate(session, conn)
# Demand 2 blocks; we store 1 (kicks one inflight transfer).
@@ -1801,8 +1773,7 @@ class TestFinishRequestServerSide:
def test_full_demand_satisfied_still_sends_success(self):
"""finish_request must not override a fully-satisfied transfer:
when remaining hits 0, success=True still fires.
"""
when remaining hits 0, success=True still fires."""
session, conn, transport = _make_session()
_activate(session, conn)
conn.enqueue(
@@ -1831,8 +1802,7 @@ class TestFinishRequestServerSide:
"""Prefiller-first: finish_request runs before the decoder's
fetch arrives. State is held until fetch, then
finalized — success=True if all demand was matched against
available blocks, else success=False.
"""
available blocks, else success=False."""
# Case A: all demand satisfied by available blocks.
session, conn, transport = _make_session()
_activate(session, conn)
@@ -1893,8 +1863,7 @@ class TestFinishRequestServerSide:
"""finish_request with a stored-but-unmatched job and no inflight ->
TransferDoneMsg(success=False) AND deferred
StoreResult(success=False) for the submit_store'd job, instead of
the 30s _STORE_TIMEOUT_S path.
"""
the 30s _STORE_TIMEOUT_S path."""
session, conn, _ = _make_session()
_activate(session, conn)
# Decoder demanded b"demand"; we never stored it.
@@ -1930,8 +1899,7 @@ class TestFinishRequestServerSide:
def test_finish_request_remaining_zero_emits_success_via_inflight(self):
"""Deferred-via-inflight path: finish_request with inflight, last
transfer drains remaining to 0 -> TransferDoneMsg(success=True)
AND StoreResult(success=True).
"""
AND StoreResult(success=True)."""
session, conn, transport = _make_session()
_activate(session, conn)
conn.enqueue(
@@ -2279,8 +2247,7 @@ class TestDisconnect:
def test_send_failure_marks_connection_dead(self):
"""A raising send must mark the connection dead, not silently drop
the message — otherwise the session lingers alive, is never reaped,
and in-flight lookups/loads toward the dead peer hang forever.
"""
and in-flight lookups/loads toward the dead peer hang forever."""
session, conn, _ = _make_session()
_activate(session, conn)
assert session.alive
@@ -2293,8 +2260,7 @@ class TestDisconnect:
def test_close_surfaces_inflight_lookups(self):
"""close() reports kv_request_ids whose symmetric-P2P probe is still
unresolved; resolved probes are not reported (their answer is in).
"""
unresolved; resolved probes are not reported (their answer is in)."""
session, conn, _ = _make_session()
_activate(session, conn)
@@ -2402,8 +2368,7 @@ class TestDispatchErrorHandling:
def test_value_error_disconnects_on_first_occurrence(self):
"""A FetchMsg that fails validate() raises ValueError and must
terminate the session immediately, with a DisconnectMsg sent.
"""
terminate the session immediately, with a DisconnectMsg sent."""
session, conn, _ = _make_session()
_activate(session, conn)
# length mismatch → FetchMsg.validate raises ValueError
@@ -2430,8 +2395,7 @@ class TestDispatchErrorHandling:
def test_internal_error_does_not_disconnect_once(self):
"""A non-ValueError raised by a handler is treated as an internal
bug: counter increments, session stays alive on a single hit.
"""
bug: counter increments, session stays alive on a single hit."""
session, conn, _ = _make_session()
_activate(session, conn)
@@ -2454,8 +2418,7 @@ class TestDispatchErrorHandling:
def test_internal_error_threshold_disconnects(self):
"""Once consecutive non-protocol errors hit the threshold, the
session tears down via _protocol_error.
"""
session tears down via _protocol_error."""
session, conn, _ = _make_session()
_activate(session, conn)
@@ -2479,8 +2442,7 @@ class TestDispatchErrorHandling:
def test_internal_error_counter_resets_on_success(self):
"""A successful dispatch between errors prevents the threshold
from being reached.
"""
from being reached."""
session, conn, _ = _make_session()
_activate(session, conn)
@@ -2523,8 +2485,7 @@ class TestInflightPerReqInvariant:
def test_invariant_holds_through_lifecycle(self):
"""Run a full submit→complete sequence for two concurrent
kv_request_ids and assert the counter matches `_inflight` at
every observable step, including the empty-after-finish case.
"""
every observable step, including the empty-after-finish case."""
session, conn, transport = _make_session()
_activate(session, conn)
@@ -2589,8 +2550,7 @@ class TestInflightPerReqInvariant:
"""Populate many inflight xfers across many ids; lookup must
match the actual presence in `_inflight` for both hits and
misses. The whole point of the counter is that this lookup is
constant-time, but we assert correctness, not timing.
"""
constant-time, but we assert correctness, not timing."""
session, _, _ = _make_session()
for kv_id_idx in range(100):
kv_id = f"req-{kv_id_idx}"
@@ -264,8 +264,7 @@ class TestAsyncLookupManager:
subsequent lookup then returns False (MISS) directly, WITHOUT enqueuing
a fresh batch_lookup — that is what makes the request unable to loop.
The entry is retained (as False) not dropped, so cleanup()'s reverse
index stays consistent; an unknown key is a no-op.
"""
index stays consistent; an unknown key is a no-op."""
mgr = InMemoryLookupManager(existing_keys={_key(1), _key(2)})
ctx = _ctx("reqA")
assert mgr.lookup(_key(1), ctx) is None
@@ -305,8 +304,7 @@ class TestAsyncLookupManager:
receive at most one result per key. Normal operation resolves a key a
single time; a second result for an already-decided key trips the assert
that guards the invariant (a silent overwrite could flip a corrected
miss back to True and reopen the failed-load livelock).
"""
miss back to True and reopen the failed-load livelock)."""
mgr = InMemoryLookupManager(existing_keys={_key(1)})
ctx = _ctx("reqA")
+8 -16
View File
@@ -369,8 +369,7 @@ def test_shutdown_discards_pending_tasks(fs_tier):
@pytest.mark.parametrize("use_c_ext", [True, False])
def test_store_load_data_integrity(fs_tier, monkeypatch, use_c_ext, batch_size):
"""Data written by store must be exactly recovered by load, for batches
of any size -- including the empty batch.
"""
of any size -- including the empty batch."""
import vllm.v1.kv_offload.tiering.fs.io as io_mod
if use_c_ext and not io_mod._HAS_FSIO_C:
@@ -467,8 +466,7 @@ def test_wait_idle_blocks_until_tasks_complete():
def test_batch_lookup_c_extension(tmp_path):
"""Validates batch_lookup_C: empty, single, all-existing, all-missing,
mixed ordering, and input type validation.
"""
mixed ordering, and input type validation."""
try:
from vllm.fs_io_C import batch_lookup as batch_lookup_C
except ImportError:
@@ -532,8 +530,7 @@ def test_batch_lookup_dispatch(fs_tier, monkeypatch, use_c_ext):
@pytest.mark.parametrize("use_c_ext", [True, False])
def test_out_of_bounds_block_id_smoke(fs_tier, monkeypatch, use_c_ext):
"""Smoke test: a block id beyond the primary tensor's block count must
fail the job, for both the C extension and the Python fallback.
"""
fail the job, for both the C extension and the Python fallback."""
import vllm.v1.kv_offload.tiering.fs.io as io_mod
if use_c_ext and not io_mod._HAS_FSIO_C:
@@ -615,8 +612,7 @@ def test_batched_partial_load_failure_keeps_loaded_blocks(
loaded before it (#50321). Corrupt the LAST block: the earlier blocks load
fine, so the job reports successful_keys for them and marks only the failed
tail a miss. The earlier keys stay HIT — including for the same request —
while the corrupt block stays a MISS (its file was removed).
"""
while the corrupt block stays a MISS (its file was removed)."""
import vllm.v1.kv_offload.tiering.fs.io as io_mod
if use_c_ext and not io_mod._HAS_FSIO_C:
@@ -668,8 +664,7 @@ def test_batched_load_first_block_fails_marks_whole_batch(
):
"""When the FIRST block fails, nothing loaded before it: the job reports no
successful_keys (None) and the whole batch is marked a miss for the
request.
"""
request."""
import vllm.v1.kv_offload.tiering.fs.io as io_mod
if use_c_ext and not io_mod._HAS_FSIO_C:
@@ -700,8 +695,7 @@ def test_transient_load_failure_leaves_file(fs_tier, monkeypatch, use_c_ext):
"""A transient host error (here ELOOP on open) is NOT a short read: the job
fails but the block file must survive untouched, on both the C and Python
paths. Deleting on a transient error would turn a passing hiccup into
permanent data loss.
"""
permanent data loss."""
import vllm.v1.kv_offload.tiering.fs.io as io_mod
if use_c_ext and not io_mod._HAS_FSIO_C:
@@ -805,8 +799,7 @@ def test_mixed_job_results_emit_event_only_for_successful_job(
fs_tier_with_events, monkeypatch
):
"""With a failed and a successful store job in flight, exactly one event
is emitted and its keys belong to the successful job.
"""
is emitted and its keys belong to the successful job."""
import vllm.v1.kv_offload.tiering.fs.manager as mgr_mod
tier = fs_tier_with_events
@@ -891,8 +884,7 @@ def test_events_require_global_kv_events_flag(tmp_path):
def test_cascade_store_emits_fs_event_through_tiering_manager(tmp_path):
"""A GPU->CPU->fs cascade surfaces the tier-owned FS stored event via the
TieringOffloadingManager's aggregated take_events().
"""
TieringOffloadingManager's aggregated take_events()."""
from vllm.v1.kv_offload.tiering.manager import (
CPUPrimaryTierOffloadingManager,
TieringOffloadingManager,
+2 -4
View File
@@ -346,8 +346,7 @@ class TestMockObjTierBasic:
the scheduler would re-issue the same doomed promotion every step for
the life of the request. The mark is served from cache with no
re-probe, so even though the mock object is still 'present' the SAME
request now resolves to MISS.
"""
request now resolves to MISS."""
ctx = ReqContext(req_id="obj-livelock")
self.tier.submit_store(make_job(1, [key(1)], [0]))
assert all(r.success for r in drain(self.tier))
@@ -651,8 +650,7 @@ class TestObjTierKVEvents:
def test_mixed_job_results_emit_event_only_for_successful_job(self):
"""With a failed and a successful store job resolving in the same
poll, exactly one event is emitted and its keys belong to the
successful job.
"""
successful job."""
original = self.agent.check_xfer_state
self.agent.check_xfer_state = lambda h: "ERR" if h._id == 0 else original(h)
self.tier.submit_store(make_job(1, [key(1)], [0])) # handle 0: fails
@@ -273,8 +273,7 @@ _CASCADE_SUPPLY_BEHAVIOR = {
def test_every_lookup_result_has_a_declared_cascade_behavior():
"""A new LookupResult must state what the cascade does with it, rather
than falling into whichever branch happens to catch it.
"""
than falling into whichever branch happens to catch it."""
assert set(_CASCADE_SUPPLY_BEHAVIOR) == set(LookupResult)
@@ -321,8 +320,7 @@ class TestTieringOffloadingManager:
def test_failed_promotion_finalizes_primary_with_failure(self, manager_setup):
"""A failed promotion still finalizes the primary slots with
success=False; the tier corrects its own verdict.
"""
success=False; the tier corrects its own verdict."""
from unittest.mock import patch
from vllm.v1.kv_offload.tiering.base import JobResult
@@ -1093,8 +1091,7 @@ class TestTieringOffloadingManager:
def test_on_new_request_lifecycle(self, manager_setup):
"""Policy defaults to CHUNK_LEVEL, escalates when a tier requests it,
and is cleaned up on on_request_finished.
"""
and is cleaned up on on_request_finished."""
# Default: all tiers return CHUNK_LEVEL
ctx = ReqContext(req_id="req_policy_lifecycle")
result = self.manager.on_new_request(ctx)
@@ -1171,8 +1168,7 @@ class TestTieringOffloadingManager:
def _make_request_level_request(self, req_id: str) -> ReqContext:
"""Start a request for which tier1 asks for request-level offloading,
with tier1's submit_store wrapped so the cascade can be observed.
"""
with tier1's submit_store wrapped so the cascade can be observed."""
self.secondary_tier1.on_new_request = lambda req_context: (
RequestOffloadingContext(policy=OffloadPolicy.REQUEST_LEVEL)
)
@@ -1194,8 +1190,7 @@ class TestTieringOffloadingManager:
def test_cascade_rejects_retry_from_primary(self, manager_setup):
"""RETRY would be parked with no guarantee of ever draining, so the
cascade refuses it. No real primary tier returns it, so this is the one
disposition that has to be forced.
"""
disposition that has to be forced."""
keys = to_keys(range(3))
self._start_request()
assert self.manager.prepare_store(keys, _CTX) is not None
@@ -1210,8 +1205,7 @@ class TestTieringOffloadingManager:
def _start_in_flight_primary_write(self, keys: list[OffloadKey]) -> ReqContext:
"""Leave a GPU->primary write for `keys` open, so they look present to
prepare_store but are not yet readable.
"""
prepare_store but are not yet readable."""
writer_ctx = ReqContext(req_id="req_writer")
self._start_request(writer_ctx)
assert self.manager.prepare_store(keys, writer_ctx) is not None
@@ -1220,8 +1214,7 @@ class TestTieringOffloadingManager:
def test_cascade_defers_keys_whose_primary_write_is_in_flight(self, manager_setup):
"""A key another request is still writing must reach the peer once the
write lands, not be dropped: prepare_store already counts it as stored
and the scheduler advances past its chunk, so nothing re-offers it.
"""
and the scheduler advances past its chunk, so nothing re-offers it."""
keys = to_keys(range(3))
writer_ctx = self._start_in_flight_primary_write(keys)
@@ -1238,8 +1231,7 @@ class TestTieringOffloadingManager:
def test_deferred_cascade_holds_request_from_finalization(self, manager_setup):
"""A request that finishes with keys still deferred must stay alive, or
its tiers are torn down before the peer is ever served.
"""
its tiers are torn down before the peer is ever served."""
keys = to_keys(range(3))
writer_ctx = self._start_in_flight_primary_write(keys)
@@ -1262,8 +1254,7 @@ class TestTieringOffloadingManager:
self, manager_setup
):
"""A failed write frees the chunk, so the deferred key resolves to MISS
and the request finalizes instead of parking forever.
"""
and the request finalizes instead of parking forever."""
keys = to_keys(range(3))
writer_ctx = self._start_in_flight_primary_write(keys)
@@ -1283,8 +1274,7 @@ class TestTieringOffloadingManager:
def test_reset_cache_clears_orchestrator_state(self, manager_setup):
"""reset_cache wipes every kind of orchestrator state and resets
primary tier; pending submissions are dropped without being sent
to the secondary tier. Active request state is retained.
"""
to the secondary tier. Active request state is retained."""
# Cascade — populates primary chunks and leaves cascade jobs
# in _jobs (the synchronous example tier has already
# queued completions); reset_cache's drain loop will pick them up.
@@ -1376,8 +1366,7 @@ class TestTieringOffloadingManager:
self, manager_setup, load_tier_filter
):
"""Filter excluding secondary medium returns MISS from secondaries
even when they hold the chunk; primary is unaffected.
"""
even when they hold the chunk; primary is unaffected."""
chunks = to_keys(range(2))
# Put one chunk in primary, one only in secondary
self._start_request()
@@ -1344,8 +1344,7 @@ def test_thinking_budget_invalid_budget_rejected(invalid_budget):
def test_thinking_budget_long_thinking_section_end_marker_found_at_correct_index():
"""Test thinking budget enforced for a long thinking run,
then a natural end marker.
"""
then a natural end marker."""
h = ThinkingBudgetStateHolder(
MockReasoningConfig(), 8, 0, torch.device("cpu"), False
)
+2 -4
View File
@@ -138,8 +138,7 @@ class EntryPoints(list):
class DummyPerReqLogitsProcessor:
"""The request-level logits processor masks out all logits except the
token id identified by `target_token`
"""
token id identified by `target_token`"""
def __init__(self, target_token: int) -> None:
"""Specify `target_token`."""
@@ -158,8 +157,7 @@ class DummyPerReqLogitsProcessor:
class WrappedPerReqLogitsProcessor(AdapterLogitsProcessor):
"""Example of wrapping a fake request-level logit processor to create a
batch-level logits processor
"""
batch-level logits processor"""
def is_argmax_invariant(self) -> bool:
return False
+1 -2
View File
@@ -1241,8 +1241,7 @@ def test_mla_per_gpu_with_pipeline_parallelism():
def test_mla_model_metrics_excludes_standard_attention():
"""Test that ModelMetrics uses MLAAttentionMetrics, not AttentionMetrics,
for DeepSeek MLA models.
"""
for DeepSeek MLA models."""
hf_config = DeepseekV3Config(
hidden_size=7168,
num_attention_heads=128,
+7 -14
View File
@@ -45,8 +45,7 @@ def test_engine_log_metrics_ray(
max_tokens: int,
) -> None:
"""Simple smoke test, verifying this can be used without exceptions.
Need to start a Ray cluster in order to verify outputs.
"""
Need to start a Ray cluster in order to verify outputs."""
@ray.remote(num_gpus=1)
class EngineTestActor:
@@ -120,8 +119,7 @@ def test_sanitized_opentelemetry_name():
def _install_mock_metric(wrapper: RayPrometheusMetric) -> MagicMock:
"""Swap the wrapper's underlying Ray metric for a MagicMock while
preserving the real metric's ``_tag_keys`` (labels() reads them to
validate arity).
"""
validate arity)."""
real_metric = wrapper.metric
mock = MagicMock()
mock._tag_keys = real_metric._tag_keys
@@ -131,8 +129,7 @@ def _install_mock_metric(wrapper: RayPrometheusMetric) -> MagicMock:
def test_ray_counter_labels_returns_independent_children():
"""RayCounterWrapper.labels() must return distinct labeled children that
each carry their own tag set.
"""
each carry their own tag set."""
base = RayCounterWrapper(
name="vllm_test_finish_reason",
documentation="",
@@ -152,8 +149,7 @@ def test_ray_counter_labels_returns_independent_children():
def test_ray_counter_inc_forwards_per_child_tags():
""".inc() on a labeled counter must forward that child's tags to the
underlying Ray metric (not rely on a shared set_default_tags).
"""
underlying Ray metric (not rely on a shared set_default_tags)."""
wrapper = RayCounterWrapper(
name="vllm_test_counter_tag_forward",
documentation="",
@@ -218,8 +214,7 @@ def test_ray_histogram_labels_returns_independent_children_and_forwards_tags():
def test_ray_counter_labels_accepts_non_string_label_values():
"""RayPrometheusStatLogger passes ``str(idx)`` for engine indexes; this
covers the coercion path for any caller that passes a non-string label
value positionally.
"""
value positionally."""
wrapper = RayCounterWrapper(
name="vllm_test_nonstr_label",
documentation="",
@@ -242,8 +237,7 @@ def test_ray_counter_labels_arity_validation():
def test_unlabeled_inc_carries_replica_id():
"""Recording on an unlabeled metric must still pass ReplicaId — it's a
declared tag_key and Ray rejects updates that omit any declared key.
"""
declared tag_key and Ray rejects updates that omit any declared key."""
wrapper = RayCounterWrapper(
name="vllm_test_unlabeled_replica_id",
documentation="",
@@ -256,8 +250,7 @@ def test_unlabeled_inc_carries_replica_id():
def test_double_labels_raises():
"""labels() on an already-labeled child should raise, mirroring the
prometheus_client contract.
"""
prometheus_client contract."""
wrapper = RayCounterWrapper(
name="vllm_test_double_labels",
documentation="",
@@ -1,8 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Test that batched_count_greater_than does not trigger 0/1 specialization
recompiles when batch_size varies.
"""
recompiles when batch_size varies."""
import torch

Some files were not shown because too many files have changed in this diff Show More