[CI] Ignore ruff D209, rejoin the docstrings it split, and silence incompatible-rule warnings (#57212)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
1085b64425
commit
651a88c09a
@@ -1392,8 +1392,7 @@ def test_non_causal_backend_correctness(
|
||||
model: str,
|
||||
):
|
||||
"""Test backend's correctness with non-causal (bidirectional) decoder
|
||||
attention, as used by DFlash speculative decoding.
|
||||
"""
|
||||
attention, as used by DFlash speculative decoding."""
|
||||
|
||||
def bidirectional_mask_mod(
|
||||
b: torch.Tensor,
|
||||
|
||||
@@ -115,8 +115,7 @@ def test_mamba_layers_have_unified_interface(
|
||||
layer_class, expected_backend, expected_mamba_type
|
||||
):
|
||||
"""Test that all Mamba layers have the unified get_attn_backend
|
||||
interface.
|
||||
"""
|
||||
interface."""
|
||||
assert hasattr(layer_class, "get_attn_backend"), (
|
||||
f"{layer_class.__name__} should have get_attn_backend method"
|
||||
)
|
||||
|
||||
@@ -164,8 +164,7 @@ def apply_split_decodes_and_prefills(
|
||||
treat_short_extends_as_decodes: bool = True,
|
||||
):
|
||||
"""Helper function to apply split_decodes_and_prefills and return
|
||||
the results.
|
||||
"""
|
||||
the results."""
|
||||
device = torch.device("cpu")
|
||||
seq_lens = [10 * (i + 1) for i in range(len(query_lens))]
|
||||
common_metadata = create_common_attn_metadata(
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""The empty-shard mask fused into the DCP A2A pack kernel must be bit-exact
|
||||
with the eager ``mask_dcp_empty_shards_`` pass it replaces.
|
||||
"""
|
||||
with the eager ``mask_dcp_empty_shards_`` pass it replaces."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
@@ -495,8 +495,7 @@ def test_dspark_noncausal_sparse_mla_matches_sdpa(
|
||||
num_heads,
|
||||
):
|
||||
"""Non-causal (window ∪ block, future-pointing) per-token indices must match
|
||||
a dense SDPA reference over the same indices, for both sparse-MLA backends.
|
||||
"""
|
||||
a dense SDPA reference over the same indices, for both sparse-MLA backends."""
|
||||
_skip_if_backend_unavailable(backend_cls, kv_cache_dtype, block_size)
|
||||
|
||||
window, block, topk_width = _DSPARK_CONFIGS[config_name]
|
||||
|
||||
@@ -231,8 +231,7 @@ def test_plan_uses_state_params(monkeypatch):
|
||||
|
||||
def test_kv_lens_host_formula():
|
||||
"""Per-row host lengths: context == position + 1; capped at
|
||||
index_topk + trailing-pool remainder past the sparse threshold.
|
||||
"""
|
||||
index_topk + trailing-pool remainder past the sparse threshold."""
|
||||
builder = object.__new__(FlashInferMLASparseSM90Builder)
|
||||
builder._index_topk = 2048
|
||||
builder._index_kpool = 4
|
||||
|
||||
@@ -187,8 +187,7 @@ def test_gdn_build_classification(test_case: GDNBuildTestCase):
|
||||
def test_has_initial_state_after_reclassification():
|
||||
"""After reclassification, num_prefills > 0 so the prefill kernel path
|
||||
should compute has_initial_state. For the reclassified request with
|
||||
context_lens > 0, the corresponding entry must be True.
|
||||
"""
|
||||
context_lens > 0, the corresponding entry must be True."""
|
||||
builder = _create_gdn_builder(num_speculative_tokens=2)
|
||||
batch = BatchSpec(seq_lens=[65, 20], query_lens=[1, 3])
|
||||
meta = _build(builder, batch, num_decode_draft_tokens=[-1, 2])
|
||||
|
||||
@@ -56,8 +56,7 @@ def _ref_stable_topk_from_candidates_fp64(
|
||||
) -> torch.Tensor:
|
||||
"""Pure-PyTorch reference for the CuteDSL stable-topk selector order
|
||||
(score desc, then lowest global token id). Selects the same SET as the
|
||||
kernel; only the set is compared in tests.
|
||||
"""
|
||||
kernel; only the set is compared in tests."""
|
||||
num_rows, num_candidates = candidate_scores.shape
|
||||
device = candidate_scores.device
|
||||
select_k = min(k, num_candidates)
|
||||
@@ -122,8 +121,7 @@ def _dcp_lse_merge(
|
||||
class _FakeDCPGroup:
|
||||
"""Single-process stand-in: ``all_gather`` returns the pre-built
|
||||
concatenation of every rank's packed ``(score, global_id)`` candidates,
|
||||
mirroring the one packed all-gather the merge issues.
|
||||
"""
|
||||
mirroring the one packed all-gather the merge issues."""
|
||||
|
||||
def __init__(self, gathered_packed: torch.Tensor) -> None:
|
||||
self.gathered_packed = gathered_packed
|
||||
@@ -209,8 +207,7 @@ def _merge_local_topks_global_with_fake_dcp(
|
||||
"""Run ``_merge_dcp_topk_global`` for every rank against a faked
|
||||
all-gather and return each rank's global-index result (all ranks should
|
||||
agree). The fake pre-builds the packed candidate concatenation the merge's
|
||||
single ``all_gather`` would return.
|
||||
"""
|
||||
single ``all_gather`` would return."""
|
||||
# The merge is now CuteDSL-only (no PyTorch fallback), so it runs the real
|
||||
# Triton pack + CuteDSL selector kernels even behind the faked all-gather.
|
||||
if not current_platform.is_cuda() or not has_cutedsl():
|
||||
@@ -712,8 +709,7 @@ def test_dcp_filter_compaction_matches_reference(
|
||||
"""In-kernel compaction must, for every row, produce exactly the rank-owned
|
||||
physical slots packed into [0, valid_count) with -1 in the tail -- the same
|
||||
SET a reference filter + sort/gather produces. Uses wide rows (> BLOCK_N
|
||||
valid slots), with interior -1 gaps.
|
||||
"""
|
||||
valid slots), with interior -1 gaps."""
|
||||
device = torch.device("cuda")
|
||||
torch.manual_seed(7)
|
||||
dcp_size = 2
|
||||
@@ -871,8 +867,7 @@ def test_decode_topk_pads_surplus_with_negative_one():
|
||||
so a non-(-1) pad would be silently attended. This is the common case under
|
||||
DCP, where each rank's local seq_len = global / world is usually << topk.
|
||||
Checked on the *set* of valid indices (the kernel may order them by score,
|
||||
not position).
|
||||
"""
|
||||
not position)."""
|
||||
torch.manual_seed(0)
|
||||
device = torch.device("cuda")
|
||||
topk = 8
|
||||
@@ -918,8 +913,7 @@ def test_sparse_decode_dcp_short_context_matches_non_dcp():
|
||||
"""End-to-end DCP decode where the global seq_len < topk (so every rank's
|
||||
local top-k is surplus-padded). Exercises the kernel surplus -> merge mask
|
||||
-> global top-k -> physical localize -> LSE merge chain for the common
|
||||
short-context decode case, vs the non-DCP reference.
|
||||
"""
|
||||
short-context decode case, vs the non-DCP reference."""
|
||||
torch.manual_seed(4)
|
||||
device = torch.device("cuda")
|
||||
world = 2
|
||||
|
||||
@@ -58,8 +58,7 @@ def test_tail_backend_layout_matches_kernel_pointer_arithmetic():
|
||||
def make_tail_block_table(own_blocks, width=64):
|
||||
"""Tail-group block table as BlockTables produces it: column 0 holds the
|
||||
request's single KpoolTailManager block, the remaining columns are never
|
||||
written and stay zero.
|
||||
"""
|
||||
written and stay zero."""
|
||||
bt = torch.zeros(len(own_blocks), width, dtype=torch.int32)
|
||||
bt[:, 0] = torch.tensor(own_blocks, dtype=torch.int32)
|
||||
return bt
|
||||
@@ -67,8 +66,7 @@ def make_tail_block_table(own_blocks, width=64):
|
||||
|
||||
def legacy_generic_tail_slots(block_table, query_start_loc, positions):
|
||||
"""Reference of the generic ``_compute_slot_mappings_kernel`` arithmetic
|
||||
(block_table.py:305-313) applied to the tail group's table.
|
||||
"""
|
||||
(block_table.py:305-313) applied to the tail group's table."""
|
||||
slots = []
|
||||
for req in range(block_table.shape[0]):
|
||||
for i in range(query_start_loc[req], query_start_loc[req + 1]):
|
||||
@@ -109,8 +107,7 @@ def make_batch(per_req_positions, padded_len=None):
|
||||
|
||||
def test_legacy_generic_mapping_collapses_onto_block_zero():
|
||||
"""The bug: with the manager's 1-column block table, the generic kernel
|
||||
maps every pos >= kpool onto tail block 0, and distinct requests collide.
|
||||
"""
|
||||
maps every pos >= kpool onto tail block 0, and distinct requests collide."""
|
||||
own_blocks = [5, 9]
|
||||
per_req = [list(range(10)), list(range(12))] # prompts of len 10 and 12
|
||||
positions, qsl, _, num_actual, num_reqs = make_batch(per_req)
|
||||
@@ -140,8 +137,7 @@ def test_legacy_generic_mapping_collapses_onto_block_zero():
|
||||
def test_circular_mapping_isolates_requests():
|
||||
"""The fix: every token lands in its own request's block at pos % kpool,
|
||||
and no slot is ever shared by two different requests (slots do recur
|
||||
within a request every kpool positions -- that is the circular design).
|
||||
"""
|
||||
within a request every kpool positions -- that is the circular design)."""
|
||||
own_blocks = [5, 9]
|
||||
per_req = [list(range(10)), list(range(12))]
|
||||
positions, qsl, slot_mapping, num_actual, num_reqs = make_batch(per_req)
|
||||
@@ -169,8 +165,7 @@ def test_circular_mapping_isolates_requests():
|
||||
def test_circular_mapping_matches_generic_for_short_requests(prompt_len):
|
||||
"""For pos < kpool the generic kernel already picks the own block, so the
|
||||
two mappings agree while every position fits the request's first block
|
||||
(single-request behavior is unchanged).
|
||||
"""
|
||||
(single-request behavior is unchanged)."""
|
||||
own_blocks = [7]
|
||||
per_req = [list(range(prompt_len))]
|
||||
positions, qsl, slot_mapping, num_actual, num_reqs = make_batch(per_req)
|
||||
@@ -280,8 +275,7 @@ class TailRingMirror:
|
||||
"""Mirror of _kpool_tail_seed_kernel / _kpool_decode_update_batched_kernel
|
||||
addressing: block = tail_slot // kpool, ring offset = pos % kpool; a pool
|
||||
completing at pos reads ring slots (pool_start + s) % kpool and uses the
|
||||
current token's own K/score for the last member.
|
||||
"""
|
||||
current token's own K/score for the last member."""
|
||||
|
||||
def __init__(self, num_blocks, kpool=KPOOL):
|
||||
self.kpool = kpool
|
||||
@@ -324,8 +318,7 @@ def tail_slot_for(mapping, req, pos, own_block):
|
||||
|
||||
def run_scenario(mapping, interleave):
|
||||
"""Requests A (block 5, prompt len 9) and B (block 9, prompt len 11)
|
||||
decode concurrently; returns A's boundary pool [8, 9, 10, 11].
|
||||
"""
|
||||
decode concurrently; returns A's boundary pool [8, 9, 10, 11]."""
|
||||
ring = TailRingMirror(num_blocks=16)
|
||||
blocks = {"A": 5, "B": 9}
|
||||
prompts = {"A": 9, "B": 11}
|
||||
|
||||
@@ -28,8 +28,7 @@ def _make_vllm_config(
|
||||
block_size: int | None = None,
|
||||
):
|
||||
"""Create a minimal mock VllmConfig with only the fields the builder
|
||||
accesses, avoiding any model download / HF config inspection.
|
||||
"""
|
||||
accesses, avoiding any model download / HF config inspection."""
|
||||
speculative_config = (
|
||||
SimpleNamespace(
|
||||
num_speculative_tokens=num_speculative_tokens,
|
||||
@@ -75,8 +74,7 @@ def test_mamba_single_token_prompt_runs_as_prefill():
|
||||
def test_update_block_table_copies_block_idx_to_persistent_buffers():
|
||||
"""update_block_table() must write block_idx tensors to the current
|
||||
builder's persistent buffers, not leave them pointing to a different
|
||||
builder's buffers.
|
||||
"""
|
||||
builder's buffers."""
|
||||
block_size = 16
|
||||
max_model_len = 256
|
||||
num_reqs = 4
|
||||
@@ -180,8 +178,7 @@ def test_state_indices_tensor_d_includes_num_speculative_blocks():
|
||||
bug 1: with mamba_cache_mode='all' and speculative decoding enabled,
|
||||
the cudagraph buffer for state_indices_tensor_d must allocate the same
|
||||
per-request column count as the runtime block table, which includes
|
||||
num_speculative_blocks trailing scratch columns.
|
||||
"""
|
||||
num_speculative_blocks trailing scratch columns."""
|
||||
block_size = 16
|
||||
max_model_len = 256
|
||||
max_num_seqs = 4
|
||||
@@ -214,8 +211,7 @@ def test_block_idx_cudagraph_capture_padded_by_num_reqs():
|
||||
bug 2: with mamba_cache_mode='all' and spec decode, _update_metadata_for
|
||||
_cudagraph_capture must slice block_idx_last_{scheduled,computed}_token
|
||||
by the request count (padded_bs == num_reqs), not by num_decode_tokens.
|
||||
Past num_decodes, the slice must be zero-filled.
|
||||
"""
|
||||
Past num_decodes, the slice must be zero-filled."""
|
||||
block_size = 16
|
||||
max_model_len = 256
|
||||
max_num_seqs = 8
|
||||
@@ -295,8 +291,7 @@ def test_block_idx_prev_step_persistent_buffer_allocated():
|
||||
block_idx_last_scheduled_token_prev_step as a persistent buffer with the
|
||||
same shape as the existing block_idx_last_{scheduled,computed}_token
|
||||
buffers, so cudagraph capture records a stable pointer for the prev-step
|
||||
input anchor consumed by mamba_mixer2's input gather.
|
||||
"""
|
||||
input anchor consumed by mamba_mixer2's input gather."""
|
||||
block_size = 16
|
||||
max_model_len = 256
|
||||
max_num_seqs = 8
|
||||
@@ -324,8 +319,7 @@ def test_block_idx_prev_step_persistent_buffer_allocated():
|
||||
|
||||
def test_block_idx_prev_step_persistent_buffer_skipped_without_spec_decode():
|
||||
"""Without spec decode, the prev-step buffer is unused and must not be
|
||||
allocated — the input anchor reduces to last_computed_token.
|
||||
"""
|
||||
allocated — the input anchor reduces to last_computed_token."""
|
||||
block_size = 16
|
||||
max_model_len = 256
|
||||
max_num_seqs = 8
|
||||
@@ -349,8 +343,7 @@ def test_block_idx_prev_step_cudagraph_capture_uses_persistent_buffer():
|
||||
"""_update_metadata_for_cudagraph_capture must copy the prev-step anchor
|
||||
into the builder's persistent buffer (so cudagraph replay reads from the
|
||||
same underlying memory), pad past num_decodes with zero, and return a
|
||||
slice of the persistent buffer in the metadata.
|
||||
"""
|
||||
slice of the persistent buffer in the metadata."""
|
||||
block_size = 16
|
||||
max_model_len = 256
|
||||
max_num_seqs = 8
|
||||
|
||||
@@ -171,8 +171,7 @@ def test_glm5_flashinfer_masked_mha_routing(
|
||||
def test_concat_k_nope_k_pe_matches_torch_cat(qk_rope_head_dim):
|
||||
"""The K concat used by the MLA prefill context loop must equal torch.cat of
|
||||
k_nope with the broadcast k_pe; with no RoPE part it returns k_nope itself
|
||||
instead of allocating and copying.
|
||||
"""
|
||||
instead of allocating and copying."""
|
||||
torch.manual_seed(0)
|
||||
num_tokens, num_heads, qk_nope_head_dim = 5, 4, 256
|
||||
k_nope = torch.randn(num_tokens, num_heads, qk_nope_head_dim, dtype=torch.bfloat16)
|
||||
|
||||
@@ -134,8 +134,7 @@ def test_flash_attn_registers_warmup_only_for_fa4(
|
||||
|
||||
def test_flash_attn_prefill_backend_signature_accepts_fused_kwargs():
|
||||
"""run_prefill_new_tokens must accept out/output_scale so the direct
|
||||
(non-**kwargs) call in forward_mha type- and runtime-checks.
|
||||
"""
|
||||
(non-**kwargs) call in forward_mha type- and runtime-checks."""
|
||||
import inspect
|
||||
|
||||
params = inspect.signature(
|
||||
@@ -151,8 +150,7 @@ def test_flash_attn_prefill_backend_signature_accepts_fused_kwargs():
|
||||
|
||||
def test_mla_impl_forward_mha_accepts_output_scale():
|
||||
"""The abstract MLA impl forward_mha must carry output_scale so every
|
||||
override (and the unconditional forward_impl call) stays compatible.
|
||||
"""
|
||||
override (and the unconditional forward_impl call) stays compatible."""
|
||||
import inspect
|
||||
|
||||
from vllm.v1.attention.backend import MLAAttentionImpl
|
||||
@@ -179,8 +177,7 @@ _FUSED_FP8_SKIP = _fused_fp8_skip_reason()
|
||||
def test_fa4_fused_fp8_output_matches_post_quant(default_vllm_config):
|
||||
"""FA4's fused FP8 write (output_scale, flash-attention#135) must match the
|
||||
bf16-attention + standalone static-FP8-quant path it replaces, since
|
||||
production uses the same output_scale for both.
|
||||
"""
|
||||
production uses the same output_scale for both."""
|
||||
from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
@@ -315,8 +315,7 @@ class TestBackendValidation:
|
||||
|
||||
def test_flash_attn_accepts_glm53_flash_nope_dimensions(self):
|
||||
"""(256, 0, 256) runs the same FA kernels as GLM-5's (192, 64, 256); a
|
||||
RoPE-carrying 320-wide query does not.
|
||||
"""
|
||||
RoPE-carrying 320-wide query does not."""
|
||||
try:
|
||||
from vllm.v1.attention.backends.mla.prefill.flash_attn import (
|
||||
FlashAttnPrefillBackend,
|
||||
|
||||
@@ -257,8 +257,7 @@ def test_replayssm_write_pos(case: ReplaySSMBuildCase):
|
||||
|
||||
def test_resumed_request_differs_from_fresh():
|
||||
"""Same token count, different decode_base: fresh (base 100) -> write_pos 5,
|
||||
resumed (base 105) -> write_pos 0.
|
||||
"""
|
||||
resumed (base 105) -> write_pos 0."""
|
||||
builder = _create_replayssm_builder(16)
|
||||
batch = BatchSpec(seq_lens=[106, 106], query_lens=[1, 1])
|
||||
common = create_common_attn_metadata(batch, BLOCK_SIZE, DEVICE).replace(
|
||||
|
||||
@@ -517,8 +517,7 @@ def test_auto_selection_for_kv_connector(
|
||||
aiter_found, expected_backend, mock_vllm_config, mock_get_cdna_version
|
||||
):
|
||||
"""Auto-selection with a KV connector and AITER enabled resolves to unified attn,
|
||||
and to triton attn if AITER not enabled.
|
||||
"""
|
||||
and to triton attn if AITER not enabled."""
|
||||
from vllm.platforms.rocm import RocmPlatform
|
||||
|
||||
with patch(
|
||||
|
||||
@@ -102,8 +102,7 @@ def expected_row_seq_lens(per_req_positions, batch_size, next_n):
|
||||
|
||||
def test_legacy_flat_layout_misaligns_and_bleeds():
|
||||
"""The bug: ``positions[:n] + 1`` with padded-row coordinates reads other
|
||||
requests' positions and prefill positions.
|
||||
"""
|
||||
requests' positions and prefill positions."""
|
||||
decode_lens, per_req, positions = make_non_uniform_batch()
|
||||
batch_size = decode_lens.shape[0]
|
||||
next_n = int(decode_lens.max())
|
||||
@@ -138,8 +137,7 @@ def test_helper_uniform_layout_matches_flat_slice():
|
||||
|
||||
def test_helper_padded_layout_per_row():
|
||||
"""Non-uniform batches map every padded row to its own token's position;
|
||||
pad rows get 0 (empty tail).
|
||||
"""
|
||||
pad rows get 0 (empty tail)."""
|
||||
decode_lens, per_req, positions = make_non_uniform_batch()
|
||||
out = _decode_topk_seq_lens(positions, decode_lens, 8, 3, 4, requires_padding=True)
|
||||
expected = expected_row_seq_lens(per_req, 3, 4)
|
||||
@@ -151,8 +149,7 @@ def test_helper_padded_layout_per_row():
|
||||
def expand_tail_region(dec_seq):
|
||||
"""Run the production pure-torch expand + append pair (the fused kernel
|
||||
replicates it exactly on the identity path) and return the tail columns
|
||||
[TOPK_TOKENS, TOPK_TOKENS + KPOOL - 1).
|
||||
"""
|
||||
[TOPK_TOKENS, TOPK_TOKENS + KPOOL - 1)."""
|
||||
rows = dec_seq.shape[0]
|
||||
pool_ids = torch.arange(SELECT_K, dtype=torch.int64).expand(rows, SELECT_K)
|
||||
valid = torch.ones_like(pool_ids, dtype=torch.bool)
|
||||
@@ -166,8 +163,7 @@ def expand_tail_region(dec_seq):
|
||||
def test_tail_expansion_legacy_vs_fixed():
|
||||
"""End-to-end tail consequence: the fixed mapping appends exactly the
|
||||
request's trailing incomplete pool; the legacy flat slice drops real tail
|
||||
tokens and emits indices far past the request's own sequence.
|
||||
"""
|
||||
tokens and emits indices far past the request's own sequence."""
|
||||
decode_lens, per_req, positions = make_non_uniform_batch()
|
||||
n = 12
|
||||
fixed = _decode_topk_seq_lens(
|
||||
|
||||
@@ -909,8 +909,7 @@ def test_index_group_convert_during_piecewise_capture():
|
||||
runs in the following eager break. The convert's side stream must not
|
||||
wait on ``logical_topk_ready`` there: an event recorded inside a captured
|
||||
segment is graph-local, and an eager wait on it raises
|
||||
cudaErrorInvalidValue.
|
||||
"""
|
||||
cudaErrorInvalidValue."""
|
||||
device = torch.device(DEVICE_TYPE)
|
||||
num_tokens, num_topk, num_requests, blocks_per_req, block_size = 8, 128, 4, 4, 16
|
||||
|
||||
@@ -1049,8 +1048,7 @@ def test_triton_convert_rejects_req_id_longer_than_token_indices():
|
||||
req_id but the output is allocated like token_indices, so a full-batch
|
||||
req_id combined with an MQA-subset token_indices wrote past the end of
|
||||
the output buffer. The wrapper must reject the length mismatch instead
|
||||
of corrupting memory.
|
||||
"""
|
||||
of corrupting memory."""
|
||||
device = torch.device(DEVICE_TYPE)
|
||||
num_topk_tokens = 128
|
||||
block_size = 64
|
||||
@@ -1105,8 +1103,7 @@ def test_flashmla_forward_bf16_kv_slices_req_id_to_mqa_tokens():
|
||||
is active, forward_mqa only receives the leading decode tokens, but
|
||||
_forward_bf16_kv passed the full-batch req_id_per_token to the index
|
||||
conversion, making it write past the end of its output buffer. The call
|
||||
site must slice req_id_per_token to the MQA tokens.
|
||||
"""
|
||||
site must slice req_id_per_token to the MQA tokens."""
|
||||
device = torch.device(DEVICE_TYPE)
|
||||
num_topk_tokens = 128
|
||||
block_size = 64
|
||||
@@ -1204,8 +1201,7 @@ def test_split_prefill_chunks(seq_lens, max_buf, expected):
|
||||
)
|
||||
def test_masked_mha_workspace_fits_single_request_boundary(max_query_len, expected):
|
||||
"""A 32K prefill needs the default workspace exactly; shrinking it would
|
||||
push a supported request onto MQA.
|
||||
"""
|
||||
push a supported request onto MQA."""
|
||||
assert (
|
||||
_masked_mha_workspace_fits(
|
||||
batch_size=1,
|
||||
@@ -1263,8 +1259,7 @@ def test_is_masked_mha_available_model_dims(
|
||||
):
|
||||
"""The allow-list gates masked MHA per exact model geometry: the DeepSeek-V3.2,
|
||||
GLM-5 and NoPE GLM-5.3-Flash layouts on an SM100-family GPU with FA4 and an
|
||||
unquantized KV cache, nothing else.
|
||||
"""
|
||||
unquantized KV cache, nothing else."""
|
||||
import vllm.model_executor.layers.attention.sparse_mla_attention as mod
|
||||
|
||||
monkeypatch.setattr(
|
||||
@@ -3604,8 +3599,7 @@ def test_fp8_mixed_batch_dcp_neutralizes_empty_rows(monkeypatch):
|
||||
"""A decode row whose top-k shard holds no local candidates (all -1) has
|
||||
undefined kernel out/lse; it must come back as (0, -inf), the identity of
|
||||
the cross-rank LSE merge, or a NaN would survive the merge even at zero
|
||||
weight (0 * NaN = NaN).
|
||||
"""
|
||||
weight (0 * NaN = NaN)."""
|
||||
num_tokens, num_heads, head_dim = 3, 2, 3
|
||||
q = torch.empty(num_tokens, num_heads, head_dim, device=DEVICE_TYPE)
|
||||
local_indices = torch.tensor(
|
||||
@@ -3778,8 +3772,7 @@ def test_hisparse_fp8_prefill_gather_uses_dedicated_stream(monkeypatch):
|
||||
def test_sparse_impl_observes_repointed_indexer_buffer():
|
||||
"""The MTP proposer repoints the draft's indexer at the target model's buffer
|
||||
after the backend impl is built, so the impl must resolve the buffer per read.
|
||||
Snapshotting it in __init__ leaves the layer reading indices nothing writes.
|
||||
"""
|
||||
Snapshotting it in __init__ leaves the layer reading indices nothing writes."""
|
||||
impl = object.__new__(FlashInferMLASparseImpl)
|
||||
own = torch.zeros(4, 8, dtype=torch.int32)
|
||||
target = torch.ones(4, 8, dtype=torch.int32)
|
||||
@@ -3795,8 +3788,7 @@ def test_sparse_impl_observes_repointed_indexer_buffer():
|
||||
|
||||
def test_explicit_topk_buffer_supersedes_indexer():
|
||||
"""Backbone skip-topk layers have no indexer, and the proposer also assigns the
|
||||
shared buffer directly onto draft submodules.
|
||||
"""
|
||||
shared buffer directly onto draft submodules."""
|
||||
impl = object.__new__(FlashInferMLASparseImpl)
|
||||
indexer = SimpleNamespace(
|
||||
topk_indices_buffer=torch.zeros(2, 2, dtype=torch.int32),
|
||||
@@ -3811,8 +3803,7 @@ def test_explicit_topk_buffer_supersedes_indexer():
|
||||
|
||||
def test_sparse_mla_common_impl_resolves_buffer_lazily():
|
||||
"""Guards the whole SparseMLACommonImpl family at once: a backend that
|
||||
snapshots the buffer instead silently loses MTP buffer sharing.
|
||||
"""
|
||||
snapshots the buffer instead silently loses MTP buffer sharing."""
|
||||
assert issubclass(SparseMLACommonImpl, SharedTopkIndicesBuffer)
|
||||
assert isinstance(SparseMLACommonImpl.topk_indices_buffer, property), (
|
||||
"topk_indices_buffer must stay a lazily-resolved property"
|
||||
|
||||
@@ -198,8 +198,7 @@ def _create_nvfp4_hnd_kv_cache(
|
||||
|
||||
def _run_trtllm_integration(batch_spec, kv_cache_dtype="auto", model_name=MODEL):
|
||||
"""Run TRTLLM attention through the full FlashInfer pipeline
|
||||
and compare against an SDPA reference.
|
||||
"""
|
||||
and compare against an SDPA reference."""
|
||||
set_random_seed(42)
|
||||
device = torch.device(f"{DEVICE_TYPE}:0")
|
||||
|
||||
@@ -427,8 +426,7 @@ def _run_trtllm_integration(batch_spec, kv_cache_dtype="auto", model_name=MODEL)
|
||||
def test_trtllm_gen_full_attention_integration(batch_spec_name: str):
|
||||
"""Test TRTLLM gen-full attention through the full FlashInfer
|
||||
MetadataBuilder.build() -> FlashInferImpl.forward() pipeline,
|
||||
with real TRTLLM kernels on Blackwell.
|
||||
"""
|
||||
with real TRTLLM kernels on Blackwell."""
|
||||
_run_trtllm_integration(BATCH_SPECS[batch_spec_name])
|
||||
|
||||
|
||||
@@ -439,8 +437,7 @@ def test_trtllm_gen_full_attention_integration(batch_spec_name: str):
|
||||
@torch.inference_mode()
|
||||
def test_trtllm_gen_nvfp4_kv_integration(batch_spec_name: str):
|
||||
"""Test TRTLLM attention with nvfp4 KV cache through the full
|
||||
FlashInfer MetadataBuilder.build() -> FlashInferImpl.forward() pipeline.
|
||||
"""
|
||||
FlashInfer MetadataBuilder.build() -> FlashInferImpl.forward() pipeline."""
|
||||
_run_trtllm_integration(
|
||||
BATCH_SPECS[batch_spec_name],
|
||||
kv_cache_dtype="nvfp4",
|
||||
|
||||
@@ -2,8 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Fine-grained partial prefix-cache hits for hybrid (full attention + mamba
|
||||
"align") models: scheduler chunk splitting, partial tail registration, CoW
|
||||
on partial hits, and same-step deferral.
|
||||
"""
|
||||
on partial hits, and same-step deferral."""
|
||||
|
||||
from math import lcm
|
||||
from types import SimpleNamespace
|
||||
@@ -175,8 +174,7 @@ def test_mamba_align_split_partial_tail_schedule(dcp_world_size: int):
|
||||
"""Chunk ends with partial hits on: block-aligned chunks, one extra stop
|
||||
at the prompt's last hash boundary (registering the partial tail), then
|
||||
the remaining tokens. block=512, hash=32, prompt=10000, budget=8192:
|
||||
0 -> 8192 -> 9728 -> 9984 -> 10000.
|
||||
"""
|
||||
0 -> 8192 -> 9728 -> 9984 -> 10000."""
|
||||
block_size = 512
|
||||
scheduler_block_size = block_size * dcp_world_size
|
||||
hash_block_size = 32
|
||||
@@ -744,8 +742,7 @@ def test_hash_aligned_query_end_uses_regular_partial_tail():
|
||||
|
||||
def test_external_mamba_hit_same_block_uses_running_cow_on_continue():
|
||||
"""An external mid-block hit must become a running request even when its
|
||||
first continuation does not need another Mamba block.
|
||||
"""
|
||||
first continuation does not need another Mamba block."""
|
||||
hash_block_size = 2
|
||||
mamba_block_size = 4 * hash_block_size
|
||||
kv_cache_config = KVCacheConfig(
|
||||
@@ -817,8 +814,7 @@ def test_external_mamba_hit_same_block_uses_running_cow_on_continue():
|
||||
|
||||
def test_boundary_state_offloads_returns_cow_target():
|
||||
"""Boundary hand-offs expose aligned snapshots and the partial-tail CoW
|
||||
target, never the overwritten CoW source.
|
||||
"""
|
||||
target, never the overwritten CoW source."""
|
||||
hash_block_size = 2
|
||||
block_size = 2 * hash_block_size
|
||||
kv_cache_config = KVCacheConfig(
|
||||
@@ -1004,8 +1000,7 @@ def test_connector_finish_registers_partial_tail_before_cleanup():
|
||||
def test_boundary_state_offload_dropped_when_request_freed_before_drain():
|
||||
"""A hand-off recorded in the same scheduling pass as the request's death
|
||||
must not be drained: its release hook has already run, so draining would
|
||||
leak a pinned block.
|
||||
"""
|
||||
leak a pinned block."""
|
||||
hash_block_size = 2
|
||||
block_size = 2 * hash_block_size
|
||||
kv_cache_config = KVCacheConfig(
|
||||
@@ -1052,8 +1047,7 @@ def test_boundary_state_offload_dropped_when_request_freed_before_drain():
|
||||
|
||||
def test_boundary_state_offloads_block_aligned_prompt():
|
||||
"""A prompt ending on a block boundary registers no CoW partial tail; its
|
||||
boundary state block is handed off as a snapshot instead (once).
|
||||
"""
|
||||
boundary state block is handed off as a snapshot instead (once)."""
|
||||
hash_block_size = 2
|
||||
block_size = 2 * hash_block_size
|
||||
kv_cache_config = KVCacheConfig(
|
||||
@@ -1107,8 +1101,7 @@ def test_boundary_state_offloads_block_aligned_prompt():
|
||||
def test_truncate_computed_blocks_preserves_sparse_prefix_positions():
|
||||
"""truncate_computed_blocks slices each group by its own block size,
|
||||
keeps null placeholders in the retained prefix, and leaves the original
|
||||
lookup result untouched (pure view, no refcount changes).
|
||||
"""
|
||||
lookup result untouched (pure view, no refcount changes)."""
|
||||
hash_block_size = 2
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=24,
|
||||
@@ -1163,8 +1156,7 @@ def test_truncate_computed_blocks_preserves_sparse_prefix_positions():
|
||||
|
||||
def test_truncate_computed_blocks_allows_short_mamba_group_only():
|
||||
"""External state may replace a short Mamba hit, but other groups must
|
||||
cover the aligned local endpoint.
|
||||
"""
|
||||
cover the aligned local endpoint."""
|
||||
hash_block_size = 2
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=24,
|
||||
@@ -1306,8 +1298,7 @@ def test_hybrid_mamba_partial_tail_owner_continue_preserves_later_hit():
|
||||
def test_hybrid_mamba_moved_partial_entry_defers_same_step_hit():
|
||||
"""The owner's move re-arms the same-step guard: the moved entry is
|
||||
filled by this step's copy, and chained same-step copies read stale
|
||||
sources, so a request hitting it in the move step must be deferred.
|
||||
"""
|
||||
sources, so a request hitting it in the move step must be deferred."""
|
||||
hash_block_size = 2
|
||||
block_size = 2 * hash_block_size
|
||||
kv_cache_config = KVCacheConfig(
|
||||
@@ -1597,8 +1588,7 @@ def test_hybrid_partial_hash_truncates_full_attention_hit_length():
|
||||
|
||||
def test_cow_retained_blocks_returned_for_release():
|
||||
"""new_step_starts returns the CoW copy retentions instead of freeing
|
||||
them; the scheduler owns releasing them once the copy has run.
|
||||
"""
|
||||
them; the scheduler owns releasing them once the copy has run."""
|
||||
hash_block_size = 2
|
||||
block_size = 2 * hash_block_size
|
||||
kv_cache_config = KVCacheConfig(
|
||||
@@ -1651,8 +1641,7 @@ def test_cow_retained_blocks_returned_for_release():
|
||||
|
||||
def test_free_cow_retained_blocks_defers_until_copy_step_processed():
|
||||
"""Scheduler releases CoW retentions immediately when the copy's step has
|
||||
been processed (or deferral is off), and defers them otherwise.
|
||||
"""
|
||||
been processed (or deferral is off), and defers them otherwise."""
|
||||
from collections import deque
|
||||
|
||||
freed: list = []
|
||||
@@ -1689,8 +1678,7 @@ def test_free_cow_retained_blocks_defers_until_copy_step_processed():
|
||||
def test_full_attention_eagle_drops_one_hash_unit():
|
||||
"""With fine-grained partial hits, eagle rewinds the hit by one hash unit
|
||||
instead of a whole cache block: the tail block's KV is append-only, so it
|
||||
still covers the reduced length and stays in the hit as a partial block.
|
||||
"""
|
||||
still covers the reduced length and stays in the hit as a partial block."""
|
||||
from vllm.v1.core.block_pool import BlockPool
|
||||
from vllm.v1.core.single_type_kv_cache_manager import FullAttentionManager
|
||||
|
||||
@@ -1766,8 +1754,7 @@ def test_hybrid_partial_hit_with_eagle_stays_within_group_blocks():
|
||||
"""Regression: with eagle, the mamba group must not receive the eagle
|
||||
lookup margin — its finder never applies the drop, so it could return a
|
||||
hit past the blocks the (dropped) full-attention group covers, crashing
|
||||
the consumer's CoW with block_idx >= len(req_blocks).
|
||||
"""
|
||||
the consumer's CoW with block_idx >= len(req_blocks)."""
|
||||
hash_block_size = 2
|
||||
block_size = 2 * hash_block_size
|
||||
kv_cache_config = KVCacheConfig(
|
||||
@@ -2369,8 +2356,7 @@ def test_retention_snapshots_handed_off_with_exact_block_ids():
|
||||
"""Under sparse retention, each retained mamba boundary state block is
|
||||
handed to the connector with its exact block id. Connectors must not resolve
|
||||
align-mode state blocks positionally because the block table is not
|
||||
append-only.
|
||||
"""
|
||||
append-only."""
|
||||
hash_block_size = 2
|
||||
block_size = 4
|
||||
manager = make_kv_cache_manager(
|
||||
@@ -2411,8 +2397,7 @@ def test_snapshot_handoff_dense_default_retention():
|
||||
"""With dense (default) retention, every materialized mamba boundary
|
||||
state block is handed off — regular mamba-align + prefix-match-unit
|
||||
deployments get store-able boundary snapshots without setting a
|
||||
prefix-cache retention interval.
|
||||
"""
|
||||
prefix-cache retention interval."""
|
||||
hash_block_size = 2
|
||||
block_size = 4
|
||||
manager = make_kv_cache_manager(
|
||||
@@ -2444,8 +2429,7 @@ def test_snapshot_handoff_dense_default_retention():
|
||||
|
||||
def _run_chunked_prefill(manager, req, chunk_size, num_chunks):
|
||||
"""Prefill ``req`` one ``chunk_size`` chunk per scheduler step, yielding the
|
||||
boundary-state hand-offs offered (and claimed) at each step.
|
||||
"""
|
||||
boundary-state hand-offs offered (and claimed) at each step."""
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req)
|
||||
for step in range(num_chunks):
|
||||
manager.new_step_starts()
|
||||
@@ -2509,8 +2493,7 @@ def test_boundary_states_offered_past_prompt_for_resumed_prefill():
|
||||
``num_prompt_tokens``. A resumed request re-prefills its generated tokens
|
||||
and every group re-saves them, so filtering on the original prompt length
|
||||
would silently strip the mamba key for boundaries full attention still
|
||||
stores; only the connector knows where its save window ends.
|
||||
"""
|
||||
stores; only the connector knows where its save window ends."""
|
||||
block_size = 8
|
||||
prompt_len = block_size
|
||||
manager = make_kv_cache_manager(
|
||||
|
||||
@@ -552,8 +552,7 @@ def _create_async_pp_scheduler(
|
||||
|
||||
def _assert_ordered_subset(delivered: list[int], emitted: list[int]) -> None:
|
||||
"""Delivered tokens must be an order-preserving subset of the emitted
|
||||
tokens with no duplicates (tokens are globally unique).
|
||||
"""
|
||||
tokens with no duplicates (tokens are globally unique)."""
|
||||
it = iter(emitted)
|
||||
for token in delivered:
|
||||
assert token in it, f"token {token} delivered out of order or twice"
|
||||
@@ -563,8 +562,7 @@ def _assert_positions_consistent(req, engine: PipelinedEngine) -> None:
|
||||
"""The i-th delivered output token must be one the runner sampled for
|
||||
exactly sequence position prompt_len + i: catches a preempted request's
|
||||
stale output landing on a position the resumed request resampled (or
|
||||
vice versa), which token-stream equality alone cannot see.
|
||||
"""
|
||||
vice versa), which token-stream equality alone cannot see."""
|
||||
for i, token in enumerate(req.output_token_ids):
|
||||
expected = req.num_prompt_tokens + i
|
||||
actual = engine.emitted_position[token]
|
||||
|
||||
@@ -178,8 +178,7 @@ def _shared_layout_config():
|
||||
class TestCSALinearGrouping:
|
||||
"""A CSA + linear-attention model (sparse attention with a compressor ring,
|
||||
plus sharded GDN and one TP-replicated PLE state) goes through the generic
|
||||
packed-group path; no model-specific branch is involved.
|
||||
"""
|
||||
packed-group path; no model-specific branch is involved."""
|
||||
|
||||
@staticmethod
|
||||
def _mamba_groups(groups):
|
||||
@@ -194,8 +193,7 @@ class TestCSALinearGrouping:
|
||||
def test_replicated_state_gets_its_own_group(self):
|
||||
"""The PLE state is TP-replicated and a different size, so it must not
|
||||
share a manager group with the sharded GDN states (the NIXL worker
|
||||
requires exactly one single-layer replicated group).
|
||||
"""
|
||||
requires exactly one single-layer replicated group)."""
|
||||
groups = get_kv_cache_groups(_shared_layout_config(), _make_csa_linear_specs())
|
||||
|
||||
mamba_groups = self._mamba_groups(groups)
|
||||
@@ -248,8 +246,7 @@ class TestCSALinearGrouping:
|
||||
def test_scratch_group_survives_computed_block_truncation(self):
|
||||
"""The scratch group contributes no computed blocks, so truncating a
|
||||
lookup result must skip it: its block size is the ring capacity, which
|
||||
neither divides the hit length nor bounds the (empty) block list.
|
||||
"""
|
||||
neither divides the hit length nor bounds the (empty) block list."""
|
||||
config = _shared_layout_config()
|
||||
config.cache_config.enable_prefix_caching = True
|
||||
groups = get_kv_cache_groups(config, _make_csa_linear_specs())
|
||||
@@ -363,8 +360,7 @@ class TestCSALinearGrouping:
|
||||
per page size, which is emitted whole, or a single-layer state that
|
||||
cannot be split at all -- the mamba layers must be sized against it.
|
||||
Sizing them against a narrower bucket splits them past what the block
|
||||
already fits, spending a pool block per extra group for no saving.
|
||||
"""
|
||||
already fits, spending a pool block per extra group for no saving."""
|
||||
config = _mock_vllm_config("BLNHC")
|
||||
config.speculative_config = None
|
||||
specs = {}
|
||||
@@ -409,8 +405,7 @@ class TestSlidingWindowBucketCap:
|
||||
unbalanced 8-layer paged MLA bucket. Left whole, the SWA bucket would
|
||||
set the block width and the MLA group would fill under a third of
|
||||
every block; capped, it splits into groups no wider than the MLA
|
||||
page.
|
||||
"""
|
||||
page."""
|
||||
config = _shared_layout_config()
|
||||
specs: dict[str, KVCacheSpec] = {}
|
||||
for layer, ratio in ((2, 2), (8, 2), (14, 2), (20, 1)):
|
||||
@@ -611,8 +606,7 @@ class TestCompressorRingGroup:
|
||||
"""DeepSeek-V4.1 compressor state: a CircularBufferSpec ring (capacity
|
||||
8) beside 128-token paged MLA groups and 64-token SWA groups. The ring
|
||||
claims one block per request, never hashes, and does not disturb the
|
||||
scheduler/hash block sizes the paged groups imply.
|
||||
"""
|
||||
scheduler/hash block sizes the paged groups imply."""
|
||||
config = _shared_layout_config()
|
||||
config.cache_config.enable_prefix_caching = True
|
||||
specs: dict[str, KVCacheSpec] = {
|
||||
|
||||
@@ -367,8 +367,7 @@ def test_free_request_with_duplicate_mm_hashes():
|
||||
"""Freeing a request whose two inputs share the same mm_hash must fully
|
||||
clean up request_cached_ids. After the first free_encoder_input call,
|
||||
cached[mm_hash] becomes empty; the second call must still remove the
|
||||
remaining input_id from request_cached_ids.
|
||||
"""
|
||||
remaining input_id from request_cached_ids."""
|
||||
manager = EncoderCacheManager(cache_size=20)
|
||||
|
||||
req = MockRequest("r1", ["imgA", "imgA"], [4, 4])
|
||||
@@ -384,8 +383,7 @@ def test_free_request_with_duplicate_mm_hashes():
|
||||
|
||||
def test_duplicate_mm_hash_stays_referenced_until_last_free():
|
||||
"""`cached` holds one reference per request, not per position, so freeing
|
||||
the first of two occurrences must not release the entry.
|
||||
"""
|
||||
the first of two occurrences must not release the entry."""
|
||||
manager = EncoderCacheManager(cache_size=20)
|
||||
req = MockRequest("r1", ["imgA", "imgA"], [4, 4])
|
||||
|
||||
@@ -406,8 +404,7 @@ def test_duplicate_mm_hash_stays_referenced_until_last_free():
|
||||
def test_duplicate_mm_hash_is_not_evicted_before_last_use():
|
||||
"""An item repeated within one request (an image carried across
|
||||
conversation turns) must not become reclaimable capacity while a later
|
||||
occurrence still has to be spliced in, or the encoder recomputes it.
|
||||
"""
|
||||
occurrence still has to be spliced in, or the encoder recomputes it."""
|
||||
manager = EncoderCacheManager(cache_size=8)
|
||||
req = MockRequest("r1", ["imgA", "imgA"], [4, 4])
|
||||
other = MockRequest("r2", ["imgB"], [8])
|
||||
|
||||
@@ -2593,8 +2593,7 @@ def test_get_kv_cache_config_kpool_tail_coowns_indexer_tensor():
|
||||
"""The kpool tail parasitizes the indexer tensors instead of getting its
|
||||
own: sibling idx/tail tensors paired by layer order, zero standalone tail
|
||||
bytes, one shared block per request, and no prefix-caching leakage from
|
||||
the kpool-sized scratch group.
|
||||
"""
|
||||
the kpool-sized scratch group."""
|
||||
model_config = ModelConfig(max_model_len=8192)
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
|
||||
@@ -2665,8 +2664,7 @@ def test_get_kv_cache_config_kpool_tail_coowns_indexer_tensor():
|
||||
|
||||
def test_glm5_kpool_tail_does_not_drag_hash_block_size():
|
||||
"""The tail's kpool-sized scratch block (4 tokens) must not constrain the
|
||||
prefix-cache hash granularity: participating groups alone decide it.
|
||||
"""
|
||||
prefix-cache hash granularity: participating groups alone decide it."""
|
||||
model_config = ModelConfig(max_model_len=8192)
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
|
||||
@@ -2742,8 +2740,7 @@ def test_get_kv_cache_config_mamba_hybrid_sharing_infeasible_no_indexer():
|
||||
|
||||
def test_get_kv_cache_config_mamba_hybrid_sharing_prepadded_mamba():
|
||||
"""Platform-prepadded mamba pages (mamba_page_size_padded hint) must not
|
||||
disable slot sharing; the layout re-pads them to the MLA page.
|
||||
"""
|
||||
disable slot sharing; the layout re-pads them to the MLA page."""
|
||||
model_config = ModelConfig(max_model_len=8192)
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
|
||||
@@ -2774,8 +2771,7 @@ def test_get_kv_cache_config_mamba_hybrid_sharing_prepadded_mamba():
|
||||
def test_get_kv_cache_config_mamba_hybrid_sharing_pp_balanced_projection():
|
||||
"""Round-robin mamba grouping keeps practical PP splits balanced: every
|
||||
stage's largest projected mamba group slice fits its projected MLA
|
||||
layers, so per-stage slot tensors all have an MLA owner.
|
||||
"""
|
||||
layers, so per-stage slot tensors all have an MLA owner."""
|
||||
model_config = ModelConfig(max_model_len=8192)
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
|
||||
@@ -2819,8 +2815,7 @@ def test_get_kv_cache_config_mamba_hybrid_sharing_pp_group_count_bump(monkeypatc
|
||||
"""PP=4's default partition [11,11,12,11] leaves stage 0 with 2 MLA
|
||||
layers but a round-robin slice of 3 under the minimum 4 mamba groups;
|
||||
grouping bumps to 5 groups so every stage's projection keeps sharing
|
||||
on instead of silently falling back on stage 0.
|
||||
"""
|
||||
on instead of silently falling back on stage 0."""
|
||||
from vllm.config import ParallelConfig
|
||||
from vllm.distributed.utils import get_pp_indices
|
||||
|
||||
@@ -2869,8 +2864,7 @@ def test_get_kv_cache_config_mamba_hybrid_sharing_pp_starved_stage(monkeypatch):
|
||||
def test_get_kv_cache_config_mamba_hybrid_sharing_beats_cross_layers_flag():
|
||||
"""Hybrid slot sharing must take precedence over the experimental
|
||||
enable_cross_layers_blocks packed layout: generic packing would give the
|
||||
MLA slots a strided view, breaking the contiguous virtual split.
|
||||
"""
|
||||
MLA slots a strided view, breaking the contiguous virtual split."""
|
||||
model_config = ModelConfig(max_model_len=8192)
|
||||
kv_transfer_config = KVTransferConfig(
|
||||
kv_connector="NixlConnector",
|
||||
@@ -3172,8 +3166,7 @@ def test_multi_run_layer_compact_strides_place_hoisted_heads():
|
||||
"""A layer-compact run region is its own dense allocation: under LHBNC the head
|
||||
groups sit between the layers and the blocks, so a run's block stride is one head
|
||||
group's slice, not the whole page (regression test for setStorage out-of-bounds
|
||||
on ROCm hybrid models).
|
||||
"""
|
||||
on ROCm hybrid models)."""
|
||||
full = new_kv_cache_spec()
|
||||
swa = new_sliding_window_spec(sliding_window=full.block_size * 2)
|
||||
page = full.page_size_bytes
|
||||
@@ -3252,8 +3245,7 @@ def test_mla_with_incompatible_swa_uses_one_full_allocation_group(caplog_vllm):
|
||||
def test_hidden_states_with_tp_scales_page_size():
|
||||
"""When TP shrinks KV pages below the hidden-state per-token cost,
|
||||
get_kv_cache_groups must scale up target block sizes so that the
|
||||
common page accommodates the unsharded hidden states.
|
||||
"""
|
||||
common page accommodates the unsharded hidden states."""
|
||||
# Simulate TP=4 sharding a model with 8 KV heads → 2 per rank.
|
||||
# KV page = block_size(16) * num_kv_heads(2) * head_size(64) * dtype(2)
|
||||
# = 16 * 2 * 64 * 2 = 4096 bytes.
|
||||
@@ -4083,8 +4075,7 @@ def test_check_enough_kv_cache_memory_reserves_null_block():
|
||||
|
||||
def test_is_full_attention_spec_unwraps_uniform_type_specs():
|
||||
"""``UniformTypeKVCacheSpecs`` is not a ``FullAttentionSpec``, so callers
|
||||
scanning groups with a bare isinstance miss DeepSeek-V4-shaped configs.
|
||||
"""
|
||||
scanning groups with a bare isinstance miss DeepSeek-V4-shaped configs."""
|
||||
common = dict(num_kv_heads=1, head_size=1, dtype=torch.float32)
|
||||
full = FullAttentionSpec(block_size=4, **common)
|
||||
mla = MLAAttentionSpec(block_size=4, **common)
|
||||
@@ -4253,8 +4244,7 @@ def test_no_warning_when_draft_group_is_identified(caplog_vllm):
|
||||
|
||||
def _deepseek_v4_specs(model_version="deepseek_v4"):
|
||||
"""DeepseekV4-shaped specs: full MLA layers plus sliding-window MLA layers
|
||||
at differing page sizes, with the MTP draft layer registered last.
|
||||
"""
|
||||
at differing page sizes, with the MTP draft layer registered last."""
|
||||
return {
|
||||
"model.layers.0.self_attn.attn": new_mla_spec(),
|
||||
"model.layers.1.self_attn.attn": new_mla_spec(),
|
||||
|
||||
@@ -55,8 +55,7 @@ def _mm_feature(offset: int, length: int) -> MultiModalFeatureSpec:
|
||||
|
||||
def test_strip_covered_mm_data() -> None:
|
||||
"""Items fully inside the computed prefix lose their data; items touching
|
||||
the uncomputed region keep it; already-None data stays None.
|
||||
"""
|
||||
the uncomputed region keep it; already-None data stays None."""
|
||||
from dataclasses import replace
|
||||
|
||||
covered = _mm_feature(offset=0, length=100)
|
||||
@@ -108,8 +107,7 @@ def _mm_feature_mixed(offset: int, length: int) -> MultiModalFeatureSpec:
|
||||
|
||||
def test_strip_covered_mm_data_mrope() -> None:
|
||||
"""For M-RoPE models, covered items keep their keep_on_cpu metadata fields
|
||||
(the worker needs them to compute positions); payload fields are dropped.
|
||||
"""
|
||||
(the worker needs them to compute positions); payload fields are dropped."""
|
||||
covered = _mm_feature_mixed(offset=0, length=100)
|
||||
uncovered = _mm_feature_mixed(offset=300, length=100)
|
||||
|
||||
@@ -129,8 +127,7 @@ def test_strip_covered_mm_data_shm_address_item() -> None:
|
||||
"""SHM address descriptors must survive stripping even when covered: the
|
||||
worker needs the address to resolve the payload and to acknowledge the
|
||||
sender's writer reference. Stripping it crashed the SHM receiver cache on
|
||||
the second identical request (vllm-project/vllm#54994).
|
||||
"""
|
||||
the second identical request (vllm-project/vllm#54994)."""
|
||||
address_item = MultiModalKwargsItem(
|
||||
{
|
||||
"address": MultiModalFieldElem(data=4096, field=MultiModalBatchedField()),
|
||||
|
||||
@@ -126,8 +126,7 @@ def make_kv_cache_manager(kv_cache_config: KVCacheConfig, **kwargs) -> KVCacheMa
|
||||
"""Build a ``KVCacheManager``, deriving ``scheduler_block_size`` from the
|
||||
config (LCM of group block sizes) unless explicitly provided. This mirrors
|
||||
``resolve_kv_cache_block_sizes`` for the non-context-parallel case used by
|
||||
these tests, so callers don't have to pass it at every site.
|
||||
"""
|
||||
these tests, so callers don't have to pass it at every site."""
|
||||
kwargs.setdefault(
|
||||
"scheduler_block_size",
|
||||
lcm(*(g.kv_cache_spec.block_size for g in kv_cache_config.kv_cache_groups)),
|
||||
@@ -3330,8 +3329,7 @@ def test_kv_cache_events_with_lora(blocks_to_cache: int):
|
||||
@pytest.mark.parametrize("group_id", [0, 1, 2])
|
||||
def test_block_stored_event_group_idx(group_id: int):
|
||||
"""Test BlockStored events emitted by cache_full_blocks carry the correct
|
||||
group_idx.
|
||||
"""
|
||||
group_idx."""
|
||||
block_size = 4
|
||||
num_tokens = block_size * 2
|
||||
|
||||
@@ -3624,8 +3622,7 @@ def test_block_removed_event_group_idx(group_id: int):
|
||||
def test_emit_cached_block_events():
|
||||
"""emit_cached_block_events emits one BlockStored for already-cached
|
||||
(reused) prefix blocks, carrying the correct group_idx /
|
||||
parent_block_hash / token_ids, and without mutating block state.
|
||||
"""
|
||||
parent_block_hash / token_ids, and without mutating block state."""
|
||||
block_size = 4
|
||||
num_cached_blocks = 3
|
||||
kv_cache_group_id = 1
|
||||
@@ -3738,8 +3735,7 @@ def test_emit_cached_block_events_zero_cached():
|
||||
|
||||
def test_eagle_enabled_removes_last_block():
|
||||
"""Verify Eagle does NOT remove blocks when request
|
||||
length is divisible by block size.
|
||||
"""
|
||||
length is divisible by block size."""
|
||||
block_size = 16
|
||||
manager = make_kv_cache_manager(
|
||||
make_kv_cache_config(block_size, num_blocks=10),
|
||||
@@ -4142,8 +4138,7 @@ def test_hybrid_cache_blocks_swa_tail_window_only():
|
||||
(its right-to-left scan stops once a contiguous match is found). Blocks
|
||||
earlier in the segment can never serve a hit, so
|
||||
``HybridKVCacheCoordinator.cache_blocks`` should skip them rather than
|
||||
polluting the prefix-cache hash map.
|
||||
"""
|
||||
polluting the prefix-cache hash map."""
|
||||
block_size = 8
|
||||
# Full attn block_size=32, SWA block_size=8, sw=8 -> lcm=32.
|
||||
# tail = ceil(7/8) = 1; per_segment = 32/8 = 4.
|
||||
@@ -4214,8 +4209,7 @@ def test_hybrid_cache_blocks_clamped_to_lcm():
|
||||
Chunks past the last lcm-aligned boundary can never participate in a
|
||||
cache hit (find_longest_cache_hit always returns lcm-aligned hits), so
|
||||
caching them only pollutes the prefix-cache hash map and keeps blocks
|
||||
on the LRU list that could otherwise return to the free pool.
|
||||
"""
|
||||
on the LRU list that could otherwise return to the free pool."""
|
||||
block_size = 16
|
||||
# Full attn block_size=32, SWA block_size=16 -> lcm=32.
|
||||
kv_cache_config = KVCacheConfig(
|
||||
@@ -4350,8 +4344,7 @@ def test_hybrid_local_kv_retention_interval_aligns_in_manager():
|
||||
)
|
||||
def test_hybrid_local_kv_retention_interval_rejects_invalid(interval, expected_match):
|
||||
"""A retention interval that is negative or not a multiple of
|
||||
scheduler_block_size errors out at construction time.
|
||||
"""
|
||||
scheduler_block_size errors out at construction time."""
|
||||
block_size = 8
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=100,
|
||||
@@ -4821,8 +4814,7 @@ def test_block_lookup_cache_multi_blocks_per_key():
|
||||
def test_can_fit_full_sequence_swa_cap_admits_long_prompt():
|
||||
"""Hybrid full+SWA model with a pool sized at the startup minimum should
|
||||
admit a prompt longer than the SWA cap, because SlidingWindowManager
|
||||
recycles blocks during chunked prefill (issue #39734).
|
||||
"""
|
||||
recycles blocks during chunked prefill (issue #39734)."""
|
||||
block_size = 16
|
||||
sliding_window = 4 * block_size # 64 tokens
|
||||
max_num_batched_tokens = 8 * block_size # 128 tokens
|
||||
@@ -4917,8 +4909,7 @@ def test_full_sequence_admission_reserves_device_blocks_for_immediate_chunk(
|
||||
|
||||
def test_can_fit_full_sequence_full_attention_still_gates_oversized():
|
||||
"""The cap only loosens the SWA group; a prompt that exceeds the
|
||||
full-attention pool capacity must still be rejected.
|
||||
"""
|
||||
full-attention pool capacity must still be rejected."""
|
||||
block_size = 16
|
||||
sliding_window = 4 * block_size
|
||||
max_num_batched_tokens = 8 * block_size
|
||||
@@ -5037,8 +5028,7 @@ def _take_free_blocks(manager: KVCacheManager, num_blocks: int) -> list[KVCacheB
|
||||
without removing them. These ref_cnt==0 blocks stand in for evictable
|
||||
cache-hit blocks left behind by a previous (e.g. preempted) request, and
|
||||
sitting at the head guarantees a later group's external ``get_new_blocks``
|
||||
would contend for them on unpatched code (issue #33775).
|
||||
"""
|
||||
would contend for them on unpatched code (issue #33775)."""
|
||||
blocks: list[KVCacheBlock] = []
|
||||
head = manager.block_pool.free_block_queue.fake_free_list_head
|
||||
for _ in range(num_blocks):
|
||||
@@ -5049,8 +5039,7 @@ def _take_free_blocks(manager: KVCacheManager, num_blocks: int) -> list[KVCacheB
|
||||
|
||||
def _assert_no_double_allocation(manager: KVCacheManager, req_id: str) -> None:
|
||||
"""No physical block may be handed out twice across groups, and every
|
||||
block referenced by the request must have a live ref_cnt.
|
||||
"""
|
||||
block referenced by the request must have a live ref_cnt."""
|
||||
block_ids = manager.get_blocks(req_id).get_block_ids()
|
||||
flat = [block_id for group in block_ids for block_id in group]
|
||||
assert len(set(flat)) == len(flat), "Block IDs are not unique across groups"
|
||||
@@ -5077,8 +5066,7 @@ def _cross_group_cache_hit(
|
||||
) -> Request:
|
||||
"""Allocate ``req_id`` with a per-group local prefix hit plus external
|
||||
(connector) computed tokens, driving the coordinator's two-phase path.
|
||||
Returns the allocated request so callers can free it (e.g. to preempt).
|
||||
"""
|
||||
Returns the allocated request so callers can free it (e.g. to preempt)."""
|
||||
block_size = _two_phase_block_size(manager)
|
||||
hit_blocks = _take_free_blocks(manager, num_groups * local_blocks_per_group)
|
||||
cache_hit = KVCacheBlocks(
|
||||
@@ -5173,8 +5161,7 @@ def test_swa_free_split_keeps_cached_tail_ahead_of_scratch():
|
||||
"""Dense retention: freeing an SWA request must place its
|
||||
uncached scratch blocks at the front of the free queue (recycled first)
|
||||
and keep its cached checkpoint blocks at the back (retained for prefix
|
||||
hits). This split is always-on, independent of the retention interval.
|
||||
"""
|
||||
hits). This split is always-on, independent of the retention interval."""
|
||||
block_size = 8
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=100,
|
||||
@@ -5285,8 +5272,7 @@ def _make_pure_swa_manager(block_size, sliding_window, num_blocks=100, **kwargs)
|
||||
def test_pure_swa_retention_interval_caches_sparse_tails():
|
||||
"""Sparse retention must work for a pure-SWA single-group model, not just
|
||||
hybrid models: only the per-interval tails plus the latest replay tail are
|
||||
cached, and a replay still hits the latest replayable boundary.
|
||||
"""
|
||||
cached, and a replay still hits the latest replayable boundary."""
|
||||
block_size = 16
|
||||
manager = _make_pure_swa_manager(
|
||||
block_size, sliding_window=block_size, retention_interval=64
|
||||
@@ -5358,8 +5344,7 @@ def test_pure_swa_retention_latest_only():
|
||||
|
||||
def test_pure_swa_dense_retention_caches_all():
|
||||
"""With retention set to ``None``, a pure-SWA model keeps dense behavior:
|
||||
every block boundary is a potential hit, so all blocks are cached.
|
||||
"""
|
||||
every block boundary is a potential hit, so all blocks are cached."""
|
||||
block_size = 16
|
||||
manager = _make_pure_swa_manager(block_size, sliding_window=block_size)
|
||||
|
||||
@@ -5388,8 +5373,7 @@ def test_mamba_reachable_block_mask_sparsifies_retention():
|
||||
"""Mamba state-snapshot retention: with a configured retention interval,
|
||||
the manager keeps one cached state per interval-sized segment (plus the
|
||||
latest replay boundary) instead of a snapshot per block, which is what
|
||||
lets a small attention block_size avoid Mamba dominating the KV pool.
|
||||
"""
|
||||
lets a small attention block_size avoid Mamba dominating the KV pool."""
|
||||
from vllm.v1.core.single_type_kv_cache_manager import MambaManager
|
||||
|
||||
block_size = 16
|
||||
@@ -5427,8 +5411,7 @@ def test_mamba_reachable_block_mask_pins_shared_prefix():
|
||||
"""A Marconi-detected shared prefix (``shared_prefix_boundary``) lands before
|
||||
``num_prompt`` so the replay-boundary rule alone would drop it. The mask must
|
||||
pin the single state block ending on that boundary so sparse retention does
|
||||
not defeat cross-request shared-prefix reuse.
|
||||
"""
|
||||
not defeat cross-request shared-prefix reuse."""
|
||||
from vllm.v1.core.single_type_kv_cache_manager import MambaManager
|
||||
|
||||
block_size = 16
|
||||
@@ -5476,8 +5459,7 @@ def test_mamba_shared_prefix_survives_zero_retention():
|
||||
detection) keeps its Mamba state block cached under
|
||||
``prefix_cache_retention_interval=0``, which otherwise retains only the
|
||||
end-of-prompt replay boundary. Without this, a shared prefix (junction
|
||||
before ``num_prompt``) would be recomputed by every sharing request.
|
||||
"""
|
||||
before ``num_prompt``) would be recomputed by every sharing request."""
|
||||
block_size = 16
|
||||
|
||||
# 16-block (256-token) prompt; replay boundary is block 240 // 16 - 1 = 14.
|
||||
@@ -5519,8 +5501,7 @@ def test_mamba_shared_prefix_reuse_under_zero_retention():
|
||||
detecting request must stay reusable by a later request under
|
||||
``prefix_cache_retention_interval=0``. Without the pin the junction is
|
||||
masked out and the later request misses; with it (and under dense) the reuse
|
||||
is preserved.
|
||||
"""
|
||||
is preserved."""
|
||||
block_size = 16
|
||||
|
||||
def last_req_hit(retention, pin):
|
||||
@@ -5603,8 +5584,7 @@ def test_swa_reachable_block_mask_final_partial_segment():
|
||||
def test_swa_reachable_block_mask_pins_shared_prefix():
|
||||
"""SWA analog of the Mamba pin: the shared-prefix junction must keep the
|
||||
``need``-block sliding-window tail ending on that boundary (not a single
|
||||
block), so a windowed hit can land there under sparse retention.
|
||||
"""
|
||||
block), so a windowed hit can land there under sparse retention."""
|
||||
from vllm.v1.core.single_type_kv_cache_manager import SlidingWindowManager
|
||||
|
||||
block_size = 16
|
||||
@@ -5649,8 +5629,7 @@ def test_swa_reachable_block_mask_pins_shared_prefix():
|
||||
def test_swa_reachable_block_mask_with_dcp_scaling():
|
||||
"""DCP shards each block's KV across ranks, scaling the effective block size.
|
||||
Verify that dcp_world_size > 1 scales block size in reachability calculations,
|
||||
producing different masks than dcp_world_size=1.
|
||||
"""
|
||||
producing different masks than dcp_world_size=1."""
|
||||
from vllm.v1.core.single_type_kv_cache_manager import SlidingWindowManager
|
||||
|
||||
block_size = 16
|
||||
@@ -5709,8 +5688,7 @@ def test_swa_reachable_block_mask_sub_block_alignment_is_dense(
|
||||
multiple of the DCP-scaled block size) cannot be represented exactly. This
|
||||
happens for hybrid offloading (e.g. Gemma), where alignment_tokens is the
|
||||
full-attention chunk size and need not divide the SWA block size. The mask
|
||||
must fall back to dense (None) rather than raise.
|
||||
"""
|
||||
must fall back to dense (None) rather than raise."""
|
||||
from vllm.v1.core.single_type_kv_cache_manager import SlidingWindowManager
|
||||
|
||||
spec = SlidingWindowSpec(
|
||||
@@ -5738,8 +5716,7 @@ def test_mamba_reachable_block_mask_ignores_dcp():
|
||||
"""Mamba uses TP, not DCP: each rank holds the full recurrent state, so a
|
||||
state block spans kv_cache_spec.block_size tokens regardless of DCP. The
|
||||
mask must not scale by dcp_world_size, so retention granularity is
|
||||
identical for any DCP world size.
|
||||
"""
|
||||
identical for any DCP world size."""
|
||||
from vllm.v1.core.single_type_kv_cache_manager import MambaManager
|
||||
|
||||
spec = MambaSpec(
|
||||
@@ -5773,8 +5750,7 @@ def test_mamba_reachable_block_mask_ignores_dcp():
|
||||
|
||||
def test_mamba_reachable_block_mask_large_dcp_stays_sparse():
|
||||
"""A large DCP world size must not scale the Mamba block size, so it can
|
||||
never collapse sparse retention into dense caching.
|
||||
"""
|
||||
never collapse sparse retention into dense caching."""
|
||||
from vllm.v1.core.single_type_kv_cache_manager import MambaManager
|
||||
|
||||
spec = MambaSpec(
|
||||
@@ -5809,8 +5785,7 @@ def test_swa_shared_prefix_reuse_under_zero_retention():
|
||||
"""SWA cross-request analog: a partial shared prefix's sliding-window tail
|
||||
must stay reusable under ``prefix_cache_retention_interval=0``. Without
|
||||
the pin the junction window is masked out and a later request misses; with
|
||||
it (and under dense) reuse is preserved.
|
||||
"""
|
||||
it (and under dense) reuse is preserved."""
|
||||
block_size = 16
|
||||
|
||||
def last_req_hit(retention, pin):
|
||||
|
||||
@@ -538,8 +538,7 @@ def test_throttle_defers_inflight_prefill_chunk():
|
||||
"""DP prefill balancing throttles ALL prefill compute on a throttled step,
|
||||
not just new admissions: an in-progress (chunked) prefill already in the
|
||||
running queue is also deferred, so the step runs decode-only, while a
|
||||
separate decode keeps being scheduled.
|
||||
"""
|
||||
separate decode keeps being scheduled."""
|
||||
scheduler = create_scheduler(
|
||||
max_num_seqs=16, max_num_batched_tokens=50, enable_chunked_prefill=True
|
||||
)
|
||||
@@ -593,8 +592,7 @@ def test_throttle_capacity_bound_guard_admits():
|
||||
"""Saturation guard: if a cadence-aligned release step cannot drain the
|
||||
waiting prefill queue (it ran out of token budget), the throttle backs off on
|
||||
the next step so the backlog cannot grow into a TTFT avalanche -- prefills are
|
||||
admitted even though throttle_prefills is set.
|
||||
"""
|
||||
admitted even though throttle_prefills is set."""
|
||||
scheduler = create_scheduler(
|
||||
max_num_seqs=16, max_num_batched_tokens=200, enable_chunked_prefill=True
|
||||
)
|
||||
@@ -1293,8 +1291,7 @@ def test_pending_hisparse_reclamation_defers_preemption(monkeypatch):
|
||||
|
||||
def test_prefix_cache_query_not_inflated_by_connector_defer():
|
||||
"""The GPU prefix-cache query is recorded at admission, so a request the
|
||||
connector defers several times is counted once, not once per retry.
|
||||
"""
|
||||
connector defers several times is counted once, not once per retry."""
|
||||
num_defers_before_matching = 3
|
||||
scheduler = create_scheduler(
|
||||
enable_prefix_caching=True,
|
||||
@@ -1322,8 +1319,7 @@ def test_prefix_cache_query_not_inflated_by_connector_defer():
|
||||
|
||||
def test_preemption_re_records_prefix_cache_query():
|
||||
"""A preempted request re-enters the lookup on resume, so its recomputation
|
||||
is counted again into the preempted stats.
|
||||
"""
|
||||
is counted again into the preempted stats."""
|
||||
scheduler = create_scheduler(enable_prefix_caching=True)
|
||||
request = create_requests(num_requests=1)[0]
|
||||
scheduler.add_request(request)
|
||||
@@ -1359,8 +1355,7 @@ def test_preemption_re_records_prefix_cache_query():
|
||||
|
||||
def test_prefix_cache_stats_not_recorded_when_caching_disabled():
|
||||
"""With prefix caching off there is no local lookup, so admitting a request
|
||||
records no phantom miss.
|
||||
"""
|
||||
records no phantom miss."""
|
||||
scheduler = create_scheduler(enable_prefix_caching=False)
|
||||
for request in create_requests(num_requests=2):
|
||||
scheduler.add_request(request)
|
||||
@@ -1374,8 +1369,7 @@ def test_prefix_cache_stats_not_recorded_when_caching_disabled():
|
||||
|
||||
def test_prefix_cache_stats_counted_once_for_retried_then_scheduled_request():
|
||||
"""A real cache hit rejected once by allocate_slots and admitted on the next
|
||||
step is counted exactly once, hits included.
|
||||
"""
|
||||
step is counted exactly once, hits included."""
|
||||
block_size = 16
|
||||
scheduler = create_scheduler(
|
||||
enable_prefix_caching=True,
|
||||
@@ -3094,8 +3088,7 @@ def create_requests_with_priority(
|
||||
|
||||
def test_priority_scheduling_basic_ordering():
|
||||
"""Test that requests are scheduled in priority order
|
||||
(lower value = higher priority).
|
||||
"""
|
||||
(lower value = higher priority)."""
|
||||
scheduler = create_scheduler_with_priority()
|
||||
|
||||
# Create requests with different priorities
|
||||
@@ -3124,8 +3117,7 @@ def test_priority_scheduling_basic_ordering():
|
||||
|
||||
def test_priority_scheduling_arrival_time_tiebreaker():
|
||||
"""Test that arrival time is used
|
||||
as tiebreaker when priorities are equal.
|
||||
"""
|
||||
as tiebreaker when priorities are equal."""
|
||||
scheduler = create_scheduler_with_priority()
|
||||
|
||||
# Create requests with same priority but different arrival times
|
||||
@@ -3282,8 +3274,7 @@ def test_priority_scheduling_preemption():
|
||||
|
||||
def test_priority_scheduling_no_preemption_when_space_available():
|
||||
"""Test that preemption doesn't happen
|
||||
when there's space for new requests.
|
||||
"""
|
||||
when there's space for new requests."""
|
||||
scheduler = create_scheduler_with_priority(
|
||||
max_num_seqs=3, # Allow 3 concurrent requests
|
||||
max_num_batched_tokens=200, # Sufficient token budget
|
||||
@@ -3336,8 +3327,7 @@ def test_priority_scheduling_no_preemption_when_space_available():
|
||||
|
||||
def test_priority_scheduling_preemption_victim_selection():
|
||||
"""Test that the correct victim is selected for
|
||||
preemption based on priority and arrival time.
|
||||
"""
|
||||
preemption based on priority and arrival time."""
|
||||
# This test verifies the priority-based victim selection logic
|
||||
# by checking the waiting queue order after adding requests with different
|
||||
# priorities
|
||||
@@ -3457,8 +3447,7 @@ def test_priority_scheduling_waiting_queue_order():
|
||||
|
||||
def test_priority_scheduling_fcfs_fallback():
|
||||
"""Test that FCFS behavior is maintained when all
|
||||
requests have same priority.
|
||||
"""
|
||||
requests have same priority."""
|
||||
scheduler = create_scheduler_with_priority()
|
||||
|
||||
# Create requests with same priority but different arrival times
|
||||
@@ -3527,8 +3516,7 @@ def test_priority_scheduling_with_limited_slots():
|
||||
|
||||
def test_priority_scheduling_heap_property():
|
||||
"""Test that the waiting queue maintains heap
|
||||
property for priority scheduling.
|
||||
"""
|
||||
property for priority scheduling."""
|
||||
scheduler = create_scheduler_with_priority(
|
||||
max_num_seqs=1, # Only one request can run at a time
|
||||
)
|
||||
@@ -3753,8 +3741,7 @@ def test_priority_scheduling_preemption_and_resumption_when_out_of_kv(
|
||||
use_ec_connector, ec_role, use_v2_model_runner
|
||||
):
|
||||
"""Test that priority scheduling preempts lower priority requests
|
||||
when out of KV cache space.
|
||||
"""
|
||||
when out of KV cache space."""
|
||||
# Create scheduler with very limited memory to force preemption
|
||||
scheduler = create_scheduler_with_priority(
|
||||
max_num_seqs=2, # Allow multiple requests
|
||||
@@ -3899,8 +3886,7 @@ def test_chunked_prefill_disabled_for_encoder_decoder(
|
||||
enable_chunked_prefill: bool, is_encoder_decoder: bool, expect_enabled: bool
|
||||
) -> None:
|
||||
"""Validate that chunked prefill is appropriately disabled for
|
||||
encoder-decoder models.
|
||||
"""
|
||||
encoder-decoder models."""
|
||||
scheduler_config = SchedulerConfig(
|
||||
enable_chunked_prefill=enable_chunked_prefill,
|
||||
is_encoder_decoder=is_encoder_decoder,
|
||||
@@ -3929,8 +3915,7 @@ def _validate_chunked_prefill_settings_for_encoder_decoder(
|
||||
scheduler_config: SchedulerConfig, is_encoder_decoder: bool, expect_enabled: bool
|
||||
) -> None:
|
||||
"""Validate chunked prefill settings in the scheduler config for
|
||||
encoder-decoder models.
|
||||
"""
|
||||
encoder-decoder models."""
|
||||
assert scheduler_config.enable_chunked_prefill is expect_enabled
|
||||
if is_encoder_decoder:
|
||||
# Encoder-decoder models should automatically disable chunked multimodal
|
||||
@@ -4020,8 +4005,7 @@ def _assert_right_encoder_inputs(
|
||||
):
|
||||
"""Verify that requests/mm_hashes should (not) in scheduled encoder input
|
||||
If check_exist is False, this function returns True
|
||||
if requests are NOT in encoder inputs
|
||||
"""
|
||||
if requests are NOT in encoder inputs"""
|
||||
# Get the scheduled encoder inputs
|
||||
# NOTE: scheduled_encoder_inputs is a dictionary with request id as key
|
||||
scheduled_encoder_inputs = output.scheduled_encoder_inputs
|
||||
@@ -4182,8 +4166,7 @@ def test_ec_connector_text_only_request(use_kv_connector):
|
||||
@pytest.mark.parametrize("use_kv_connector", [False, True])
|
||||
def test_ec_connector_cache_hit_external_load(use_kv_connector):
|
||||
"""Test ec_consumer loads from external cache when hit.
|
||||
A normal basic operation for EPD disaggrgation
|
||||
"""
|
||||
A normal basic operation for EPD disaggrgation"""
|
||||
scheduler = create_scheduler(
|
||||
model="llava-hf/llava-1.5-7b-hf",
|
||||
enable_prefix_caching=True,
|
||||
@@ -4630,8 +4613,7 @@ def test_priority_scheduling_ec_connector_preemption_and_resumption(
|
||||
cache_exist, use_kv_connector
|
||||
):
|
||||
"""Test that priority scheduling preempts lower priority requests
|
||||
when out of KV cache space.
|
||||
"""
|
||||
when out of KV cache space."""
|
||||
# Create scheduler with very limited memory to force preemption
|
||||
scheduler = create_scheduler_with_priority(
|
||||
model="llava-hf/llava-1.5-7b-hf",
|
||||
@@ -5300,8 +5282,7 @@ def test_scheduler_kv_connector_stats():
|
||||
|
||||
def test_ec_connector_update_connector_output_called():
|
||||
"""Test that worker-side EC connector output is forwarded to the
|
||||
EC connector's update_connector_output hook.
|
||||
"""
|
||||
EC connector's update_connector_output hook."""
|
||||
scheduler = create_scheduler(
|
||||
model="llava-hf/llava-1.5-7b-hf",
|
||||
use_ec_connector=True,
|
||||
@@ -5506,8 +5487,7 @@ def test_variable_length_cross_attn_block_allocation():
|
||||
|
||||
def test_cross_attn_blocks_not_over_allocated():
|
||||
"""Test that cross-attention blocks are not over-allocated compared to
|
||||
what each request actually needs.
|
||||
"""
|
||||
what each request actually needs."""
|
||||
from math import ceil
|
||||
|
||||
block_size = 16
|
||||
@@ -5603,8 +5583,7 @@ def test_cross_attn_blocks_not_under_allocated():
|
||||
|
||||
def test_cross_attn_zero_blocks_without_encoder_inputs():
|
||||
"""Test that requests without encoder inputs get zero cross-attention
|
||||
blocks, even when the scheduler is configured for encoder-decoder.
|
||||
"""
|
||||
blocks, even when the scheduler is configured for encoder-decoder."""
|
||||
block_size = 16
|
||||
scheduler = _create_encoder_decoder_scheduler(block_size=block_size)
|
||||
|
||||
@@ -5709,8 +5688,7 @@ def test_free_encoder_inputs_respects_unconfirmed_placeholders():
|
||||
evicted entry and crashes the engine with "Encoder cache miss". The
|
||||
scheduler must retain the input until the *confirmed* progress
|
||||
(num_computed_tokens - num_output_placeholders) passes the range end, so
|
||||
that no pending rejection can rewind into the range.
|
||||
"""
|
||||
that no pending rejection can rewind into the range."""
|
||||
scheduler = create_scheduler(
|
||||
model="llava-hf/llava-1.5-7b-hf",
|
||||
num_speculative_tokens=3,
|
||||
@@ -5821,8 +5799,7 @@ def test_free_encoder_inputs_defers_for_eagle_lookahead():
|
||||
position so the drafter's +1 look-ahead mm-embedding gather (which reads one
|
||||
position past the target's computed range) still finds it cached. This is
|
||||
the primary mechanism that prevents the drafter "Encoder cache miss"; the
|
||||
worker-side token-embedding fallback is only a backstop.
|
||||
"""
|
||||
worker-side token-embedding fallback is only a backstop."""
|
||||
scheduler = create_scheduler(model="llava-hf/llava-1.5-7b-hf")
|
||||
# create_scheduler only builds ngram spec configs; force the eagle path that
|
||||
# _free_encoder_inputs keys off (its read-ahead deferral).
|
||||
@@ -5852,8 +5829,7 @@ def test_free_encoder_inputs_defers_for_eagle_lookahead():
|
||||
|
||||
def test_free_encoder_inputs_unchanged_without_spec_decode():
|
||||
"""Without speculative decoding, encoder inputs are freed as soon as
|
||||
num_computed_tokens passes the placeholder range, as before.
|
||||
"""
|
||||
num_computed_tokens passes the placeholder range, as before."""
|
||||
scheduler = create_scheduler(model="llava-hf/llava-1.5-7b-hf")
|
||||
mm_positions = [[PlaceholderRange(offset=50, length=100)]]
|
||||
request = create_requests(
|
||||
@@ -5884,8 +5860,7 @@ def test_encoder_cache_retained_across_preemption_and_resume():
|
||||
encoder input must pull the still-cached entry back out of `freeable`
|
||||
without scheduling a recompute, keeping the scheduler and worker
|
||||
consistent. The spec-rollback retention margin does not gate this path,
|
||||
so it is covered separately here.
|
||||
"""
|
||||
so it is covered separately here."""
|
||||
scheduler = create_scheduler(
|
||||
model="llava-hf/llava-1.5-7b-hf",
|
||||
num_speculative_tokens=3,
|
||||
@@ -5929,8 +5904,7 @@ def test_encoder_cache_recomputed_when_evicted_during_preemption():
|
||||
pressure before it resumes, the scheduler reports the mm_hash as freed
|
||||
(so the worker drops it) and a resume must schedule a recompute rather
|
||||
than assume the worker still holds it. check_and_update_cache must
|
||||
return False so the encoder input is re-scheduled.
|
||||
"""
|
||||
return False so the encoder input is re-scheduled."""
|
||||
scheduler = create_scheduler(
|
||||
model="llava-hf/llava-1.5-7b-hf",
|
||||
num_speculative_tokens=3,
|
||||
@@ -6581,8 +6555,7 @@ def _decode_ready_request(scheduler):
|
||||
|
||||
def test_update_draft_token_ids_strips_ngram_padding():
|
||||
"""ngram_gpu pads unfilled draft slots with -1; manager validation must
|
||||
strip them before grammar.validate_tokens (which otherwise raises).
|
||||
"""
|
||||
strip them before grammar.validate_tokens (which otherwise raises)."""
|
||||
scheduler = create_scheduler(num_speculative_tokens=4)
|
||||
request = _decode_ready_request(scheduler)
|
||||
|
||||
@@ -6601,8 +6574,7 @@ def test_update_draft_token_ids_strips_ngram_padding():
|
||||
|
||||
def test_update_draft_token_ids_in_output_strips_padding():
|
||||
"""Same guard on the output path; the -1 pad-back for the rejected count
|
||||
is preserved (only the input to manager validation is stripped).
|
||||
"""
|
||||
is preserved (only the input to manager validation is stripped)."""
|
||||
scheduler = create_scheduler(num_speculative_tokens=4)
|
||||
request = _decode_ready_request(scheduler)
|
||||
|
||||
|
||||
@@ -97,8 +97,7 @@ def test_num_in_flight_tokens_accounting():
|
||||
|
||||
def test_swa_free_waits_for_in_flight_step():
|
||||
"""Async: out-of-window blocks stay allocated until the step that still
|
||||
reads them has been processed.
|
||||
"""
|
||||
reads them has been processed."""
|
||||
scheduler = _create_swa_scheduler(async_scheduling=True)
|
||||
request = create_requests(
|
||||
num_requests=1, num_tokens=NUM_PROMPT_TOKENS, block_size=BLOCK_SIZE
|
||||
@@ -133,8 +132,7 @@ def test_swa_free_waits_for_in_flight_step():
|
||||
|
||||
def test_swa_free_immediate_when_sync():
|
||||
"""Sync: no in-flight step at schedule time, frees happen at the first
|
||||
decode allocation as before.
|
||||
"""
|
||||
decode allocation as before."""
|
||||
scheduler = _create_swa_scheduler(async_scheduling=False)
|
||||
request = create_requests(
|
||||
num_requests=1, num_tokens=NUM_PROMPT_TOKENS, block_size=BLOCK_SIZE
|
||||
@@ -173,8 +171,7 @@ def test_swa_admission_cap_accounts_for_overlapping_batches():
|
||||
def test_chunked_local_free_waits_for_in_flight_step():
|
||||
"""Chunked-local attention frees whole chunks left of the current one, and
|
||||
is exposed to the same load-WAR: with async scheduling those chunks must
|
||||
stay allocated until the in-flight step that still reads them settles.
|
||||
"""
|
||||
stay allocated until the in-flight step that still reads them settles."""
|
||||
scheduler = _create_chunked_scheduler(async_scheduling=True)
|
||||
request = create_requests(
|
||||
num_requests=1, num_tokens=NUM_PROMPT_TOKENS, block_size=BLOCK_SIZE
|
||||
@@ -229,8 +226,7 @@ def test_connector_finish_frees_on_settled_basis():
|
||||
is handed to a KV connector (simple CPU offload / NIXL store), must use the
|
||||
same processed-token basis. Otherwise the connector reads/hands off a block
|
||||
the still-in-flight step is optimistically counted as done with, which is
|
||||
the load-path WAR this fix closes.
|
||||
"""
|
||||
the load-path WAR this fix closes."""
|
||||
scheduler = create_scheduler(
|
||||
block_size=BLOCK_SIZE,
|
||||
async_scheduling=True,
|
||||
|
||||
@@ -80,8 +80,7 @@ def test_piecewise_capture_builds_fresh_metadata_for_both_passes():
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_breakable_tls():
|
||||
"""Defensively clear thread-local capture state between tests so a
|
||||
failure in one test can't leak "nested capture" errors into the next.
|
||||
"""
|
||||
failure in one test can't leak "nested capture" errors into the next."""
|
||||
from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphCapture
|
||||
|
||||
BreakableCUDAGraphCapture._tls.active = None
|
||||
@@ -291,8 +290,7 @@ def test_capture_replay_matches_eager_simple(cuda_capture_stream):
|
||||
def test_decorator_breaks_when_invoked_inside_capture(cuda_capture_stream):
|
||||
"""Verify @eager_break_during_capture correctly routes through
|
||||
add_eager when inside a capture context, and runs straight through
|
||||
when there's no active capture.
|
||||
"""
|
||||
when there's no active capture."""
|
||||
from vllm.compilation.breakable_cudagraph import (
|
||||
BreakableCUDAGraphCapture,
|
||||
eager_break_during_capture,
|
||||
@@ -554,8 +552,7 @@ def _mock_vllm_config():
|
||||
|
||||
def test_wrapper_falls_through_on_runtime_mode_mismatch():
|
||||
"""A PIECEWISE-scoped wrapper runs other modes eagerly, capturing
|
||||
nothing, so an outer CUDAGraphWrapper can own them.
|
||||
"""
|
||||
nothing, so an outer CUDAGraphWrapper can own them."""
|
||||
from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphWrapper
|
||||
from vllm.config import CUDAGraphMode
|
||||
from vllm.forward_context import BatchDescriptor, set_forward_context
|
||||
|
||||
@@ -275,8 +275,7 @@ def test_rms_norm_per_block_quant_batch_invariant(
|
||||
hidden_size: int, dtype: torch.dtype, seed: int
|
||||
):
|
||||
"""C++ ``rms_norm_per_block_quant`` must be batch invariant across the
|
||||
block threshold (compiled fp8 block-quant path; block pinned to 512).
|
||||
"""
|
||||
block threshold (compiled fp8 block-quant path; block pinned to 512)."""
|
||||
import vllm._custom_ops as ops
|
||||
|
||||
device = torch.device(DEVICE_TYPE)
|
||||
|
||||
@@ -547,8 +547,7 @@ async def test_dp_pause_keep_then_resume(expert_parallel: bool):
|
||||
async def test_dp_pause_keep_race_staggered_engines():
|
||||
"""Race: send pause(keep) to engine 0, then add two requests,
|
||||
then pause(keep) to engine 1. Ensures no deadlock when pause
|
||||
requests are staggered and requests arrive in between.
|
||||
"""
|
||||
requests are staggered and requests arrive in between."""
|
||||
if DP_SIZE != 2:
|
||||
pytest.skip("test_dp_pause_keep_race_staggered_engines requires DP_SIZE=2")
|
||||
|
||||
@@ -718,8 +717,7 @@ async def test_dp_pause_barrier_request_deadlock():
|
||||
async def test_dp_pause_wait_mode_drains_in_flight():
|
||||
"""mode="wait" through the DP consensus path: the pause lets the
|
||||
in-flight request run to completion while the consensus is pending,
|
||||
resolves only once drained, and the engine generates again on resume.
|
||||
"""
|
||||
resolves only once drained, and the engine generates again on resume."""
|
||||
with ExitStack() as after:
|
||||
engine = AsyncLLM.from_engine_args(_get_dp_pause_engine_args(True))
|
||||
after.callback(engine.shutdown)
|
||||
@@ -750,8 +748,7 @@ async def test_dp_pause_wait_mode_drains_in_flight():
|
||||
async def test_dp_pause_while_asleep():
|
||||
"""Pausing a sleeping DP engine: sleeping ranks skip dummy batches yet
|
||||
must still reach the pause consensus, and the completion barrier must
|
||||
be harmless on workers whose memory is unmapped.
|
||||
"""
|
||||
be harmless on workers whose memory is unmapped."""
|
||||
with ExitStack() as after:
|
||||
engine = AsyncLLM.from_engine_args(_get_dp_pause_engine_args(True))
|
||||
after.callback(engine.shutdown)
|
||||
@@ -787,8 +784,7 @@ async def test_dp_pause_completion_implies_device_idle(
|
||||
"""A resolved pause future promises an idle device. Inflate the dummy
|
||||
batches the pause consensus manufactures so any work the pause fails to
|
||||
wait on stays visible, then require a quiet worker stream the moment
|
||||
pause_generation() returns.
|
||||
"""
|
||||
pause_generation() returns."""
|
||||
with ExitStack() as after:
|
||||
# The probes below ship functions through collective_rpc.
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
|
||||
@@ -24,8 +24,7 @@ TP_SIZE = int(os.getenv("TP_SIZE", "1"))
|
||||
|
||||
class ExternalLBServerManager:
|
||||
"""Manages data parallel vLLM server instances for external
|
||||
load balancer testing.
|
||||
"""
|
||||
load balancer testing."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -30,8 +30,7 @@ DP_SIZE_LOCAL = DP_SIZE // NUM_NODES # 2 ranks per node
|
||||
class HybridLBServerManager:
|
||||
"""Manages hybrid data parallel vLLM server instances where each node
|
||||
runs a single logical API server that balances requests only to the
|
||||
DP engines running on that same node.
|
||||
"""
|
||||
DP engines running on that same node."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -107,8 +107,7 @@ async def _run_request_bursts(
|
||||
|
||||
class MultinodeInternalLBServerManager:
|
||||
"""Manages multi-node data parallel vLLM server instances for internal
|
||||
load balancer testing using --headless mode.
|
||||
"""
|
||||
load balancer testing using --headless mode."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -242,8 +241,7 @@ class MultinodeInternalLBServerManager:
|
||||
|
||||
class APIOnlyServerManager:
|
||||
"""Manages API-only server (Node 0) and headless engines server (Node 1)
|
||||
for testing separated API server and engine configuration.
|
||||
"""
|
||||
for testing separated API server and engine configuration."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -645,8 +643,7 @@ async def test_api_only_multinode_dp_completion_streaming(
|
||||
model_name: str,
|
||||
) -> None:
|
||||
"""Test API-only server streaming with all engines on separate
|
||||
headless server.
|
||||
"""
|
||||
headless server."""
|
||||
prompt = "What is an LLM?"
|
||||
|
||||
async def make_streaming_request():
|
||||
|
||||
@@ -82,8 +82,7 @@ async def _generate(engine: AsyncLLM, prompt: str, max_tokens: int) -> int:
|
||||
@pytest.mark.parametrize("async_scheduling", [True, False])
|
||||
async def test_pp_dp_v2_low_concurrency(async_scheduling: bool):
|
||||
"""A single in-flight request at a time, repeated, to exercise the
|
||||
PP slot ring under empty batches between decodes.
|
||||
"""
|
||||
PP slot ring under empty batches between decodes."""
|
||||
with ExitStack() as after:
|
||||
engine = AsyncLLM.from_engine_args(_engine_args(async_scheduling))
|
||||
after.callback(engine.shutdown)
|
||||
@@ -97,8 +96,7 @@ async def test_pp_dp_v2_low_concurrency(async_scheduling: bool):
|
||||
@pytest.mark.parametrize("async_scheduling", [True, False])
|
||||
async def test_pp_dp_v2_mid_concurrency(async_scheduling: bool):
|
||||
"""64 concurrent requests, staggered, to exercise the steady-state
|
||||
DP all-reduce + PP slot-ring path.
|
||||
"""
|
||||
DP all-reduce + PP slot-ring path."""
|
||||
with ExitStack() as after:
|
||||
engine = AsyncLLM.from_engine_args(_engine_args(async_scheduling))
|
||||
after.callback(engine.shutdown)
|
||||
@@ -114,8 +112,7 @@ async def test_pp_dp_v2_mid_concurrency(async_scheduling: bool):
|
||||
@pytest.mark.asyncio
|
||||
async def test_pp_dp_v2_abort_mid_decode():
|
||||
"""Cancel half the in-flight requests mid-stream and confirm the
|
||||
engine survives the abort storm.
|
||||
"""
|
||||
engine survives the abort storm."""
|
||||
with ExitStack() as after:
|
||||
engine = AsyncLLM.from_engine_args(_engine_args(async_scheduling=True))
|
||||
after.callback(engine.shutdown)
|
||||
@@ -155,8 +152,7 @@ async def test_pp_dp_v2_abort_mid_decode():
|
||||
@pytest.mark.asyncio
|
||||
async def test_pp_dp_v2_pause_resume():
|
||||
"""Pause an engine with a request in flight, then resume and confirm
|
||||
new requests still work.
|
||||
"""
|
||||
new requests still work."""
|
||||
with ExitStack() as after:
|
||||
engine = AsyncLLM.from_engine_args(_engine_args(async_scheduling=True))
|
||||
after.callback(engine.shutdown)
|
||||
|
||||
@@ -48,8 +48,7 @@ def test_without_spec_decoding(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""Test consistency of combos of async scheduling, preemption,
|
||||
uni/multiproc executor, prefill chunking.
|
||||
"""
|
||||
uni/multiproc executor, prefill chunking."""
|
||||
struct_outputs = StructuredOutputsParams(json=sample_json_schema)
|
||||
test_sampling_params: list[dict[str, Any]] = [
|
||||
dict(),
|
||||
@@ -205,8 +204,7 @@ def run_tests(
|
||||
test_sampling_params: list[dict[str, Any]],
|
||||
):
|
||||
"""Test consistency of combos of async scheduling, preemption,
|
||||
uni/multiproc executor with spec decoding.
|
||||
"""
|
||||
uni/multiproc executor with spec decoding."""
|
||||
# Flex attention supports float32.
|
||||
attention_config = {"backend": "FLEX_ATTENTION"}
|
||||
|
||||
|
||||
@@ -72,8 +72,7 @@ def test_backend_per_kind_splits_groups(backend_per_kind, monkeypatch):
|
||||
)
|
||||
def test_backend_per_kind_overrides_global_backend(monkeypatch):
|
||||
"""A per-kind entry wins over the global ``backend`` for its kind; other
|
||||
kinds fall back to the global backend.
|
||||
"""
|
||||
kinds fall back to the global backend."""
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
llm = LLM(
|
||||
model=MODEL,
|
||||
|
||||
@@ -34,8 +34,7 @@ def test_decoder_max_context_length_validation(
|
||||
max_tokens: int,
|
||||
) -> None:
|
||||
"""Check vLLM decoder model input validation for edge cases where
|
||||
the prompt length is (almost) equal to the max model length.
|
||||
"""
|
||||
the prompt length is (almost) equal to the max model length."""
|
||||
prompt_ids = [[43] * prompt_len]
|
||||
|
||||
with vllm_runner(
|
||||
@@ -72,8 +71,7 @@ def test_auto_fit_max_model_len_rejects_oversized_input(
|
||||
"""When max_model_len='auto' and KV cache memory is very limited,
|
||||
the engine auto-fits max_model_len to a small value. The frontend
|
||||
must see this reduced value and reject prompts that exceed it,
|
||||
rather than accepting them and hanging.
|
||||
"""
|
||||
rather than accepting them and hanging."""
|
||||
# Use a small KV cache budget to force auto-fit to a small
|
||||
# max_model_len. Pin block_size=16 so the budget is independent
|
||||
# of the platform's default block size. One block for this model is
|
||||
|
||||
@@ -172,8 +172,7 @@ def llm_v1():
|
||||
|
||||
def test_min_tokens_with_terminal_structured_output(llm_v1: LLM):
|
||||
"""A grammar whose only legal continuation is EOS must stop under
|
||||
min_tokens instead of sampling outside the grammar.
|
||||
"""
|
||||
min_tokens instead of sampling outside the grammar."""
|
||||
params = SamplingParams(
|
||||
temperature=GREEDY,
|
||||
min_tokens=20,
|
||||
|
||||
@@ -7,8 +7,7 @@ slightly different kernel/collective states (measured: same-state boots are
|
||||
bitwise identical across the two modes; cross-state boots differ by up to
|
||||
~0.25 logprob and can flip a near-tie token). A real sharding bug — rows
|
||||
routed to the wrong request, misaligned gathers — produces many-nats
|
||||
divergence and scrambled top-k sets, far outside these bounds.
|
||||
"""
|
||||
divergence and scrambled top-k sets, far outside these bounds."""
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -107,8 +106,7 @@ def _assert_logprob_dicts_close(ref_lps, out_lps, what: str) -> None:
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
def test_sharded_sampling_outputs_match(monkeypatch: pytest.MonkeyPatch):
|
||||
"""Generation, logprobs, and prompt logprobs match between batch-sharded
|
||||
sampling and the replicated fallback, up to measured boot-state noise.
|
||||
"""
|
||||
sampling and the replicated fallback, up to measured boot-state noise."""
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
# Required for the collective_rpc mode check below.
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
|
||||
@@ -11,8 +11,7 @@ from ...utils import compute_acceptance_len, get_test_prompts
|
||||
@single_gpu_only
|
||||
def test_synthetic_acceptance_rate(vllm_runner):
|
||||
"""Verify that synthetic rejection sampling produces an acceptance
|
||||
length close to the requested mean acceptance length.
|
||||
"""
|
||||
length close to the requested mean acceptance length."""
|
||||
num_spec_tokens = 3
|
||||
expected_acceptance_len = 1.875
|
||||
tolerance = 0.15
|
||||
|
||||
@@ -167,8 +167,7 @@ def test_draft_model_tensor_parallelism(vllm_runner):
|
||||
@multi_gpu_only(num_gpus=2)
|
||||
def test_draft_model_engine_args_tensor_parallelism():
|
||||
"""Ensure the vllm_config for the draft model is created correctly,
|
||||
and independently of the target model (quantization, TP, etc.)
|
||||
"""
|
||||
and independently of the target model (quantization, TP, etc.)"""
|
||||
engine_args = EngineArgs(
|
||||
model="Qwen/Qwen3-1.7B-FP8", # <<< tgt quantized
|
||||
tensor_parallel_size=2,
|
||||
@@ -199,8 +198,7 @@ def test_draft_model_engine_args_tensor_parallelism():
|
||||
|
||||
def _apply_draft_moe_backend(vllm_config: VllmConfig) -> VllmConfig:
|
||||
"""Replicate SpecDecodeBaseProposer._create_draft_vllm_config logic
|
||||
so we can test it without instantiating a full proposer.
|
||||
"""
|
||||
so we can test it without instantiating a full proposer."""
|
||||
spec_cfg = vllm_config.speculative_config
|
||||
if spec_cfg.moe_backend is not None:
|
||||
return replace(
|
||||
@@ -225,8 +223,7 @@ def _platform_moe_backend(monkeypatch: pytest.MonkeyPatch) -> MoEBackend:
|
||||
|
||||
def test_draft_model_moe_backend_override(monkeypatch: pytest.MonkeyPatch):
|
||||
"""When moe_backend is set in speculative_config, the draft VllmConfig
|
||||
should use it while the target keeps its own setting.
|
||||
"""
|
||||
should use it while the target keeps its own setting."""
|
||||
target_moe_backend = _platform_moe_backend(monkeypatch)
|
||||
engine_args = EngineArgs(
|
||||
model="Qwen/Qwen3-1.7B",
|
||||
@@ -251,8 +248,7 @@ def test_draft_model_moe_backend_override(monkeypatch: pytest.MonkeyPatch):
|
||||
|
||||
def test_draft_model_moe_backend_inherits_target(monkeypatch: pytest.MonkeyPatch):
|
||||
"""When moe_backend is not set in speculative_config, the draft should
|
||||
inherit the target's moe_backend.
|
||||
"""
|
||||
inherit the target's moe_backend."""
|
||||
target_moe_backend = _platform_moe_backend(monkeypatch)
|
||||
engine_args = EngineArgs(
|
||||
model="Qwen/Qwen3-1.7B",
|
||||
@@ -275,8 +271,7 @@ def test_draft_model_moe_backend_inherits_target(monkeypatch: pytest.MonkeyPatch
|
||||
|
||||
def test_draft_model_moe_backend_default_auto():
|
||||
"""When neither target nor draft set moe_backend explicitly, both should
|
||||
default to 'auto'.
|
||||
"""
|
||||
default to 'auto'."""
|
||||
engine_args = EngineArgs(
|
||||
model="Qwen/Qwen3-1.7B",
|
||||
tensor_parallel_size=1,
|
||||
@@ -297,8 +292,7 @@ def test_draft_model_moe_backend_default_auto():
|
||||
|
||||
def test_draft_model_engine_args_rejects_invalid_tp_argname():
|
||||
"""The user should pass "draft_tensor_parallel_size" rather than
|
||||
"tensor_parallel_size". We enforce this with validation.
|
||||
"""
|
||||
"tensor_parallel_size". We enforce this with validation."""
|
||||
engine_args = EngineArgs(
|
||||
model="Qwen/Qwen3-1.7B",
|
||||
tensor_parallel_size=1,
|
||||
@@ -315,8 +309,7 @@ def test_draft_model_engine_args_rejects_invalid_tp_argname():
|
||||
|
||||
def assert_draft_model_correctness(args: ArgsTest, vllm_runner):
|
||||
"""Compare the outputs using and not using speculative decoding.
|
||||
In the greedy decoding case, the outputs must match EXACTLY.
|
||||
"""
|
||||
In the greedy decoding case, the outputs must match EXACTLY."""
|
||||
test_prompts: list[Messages] = get_messages(
|
||||
dataset=args.dataset, n=args.num_prompts
|
||||
)
|
||||
|
||||
@@ -28,8 +28,7 @@ def test_mtp_sharded_sampling_equivalence(monkeypatch: pytest.MonkeyPatch):
|
||||
sampling under MTP spec decoding: the collectives move the same logits
|
||||
bytes, Gumbel keys derive from (request slot, position, seed), and slot
|
||||
assignment is rank-deterministic. Both runs here are spec decode with
|
||||
identical math, so outputs must match exactly.
|
||||
"""
|
||||
identical math, so outputs must match exactly."""
|
||||
tp_size = 2
|
||||
_skip_if_insufficient_gpus_for_tp(tp_size)
|
||||
model_name = "Qwen/Qwen3.5-0.8B-Base"
|
||||
|
||||
@@ -67,8 +67,7 @@ def test_mtp_speculative_mixed_batch_short_prefill(
|
||||
vllm_runner, model_name, enable_prefix_caching
|
||||
):
|
||||
"""Test to ensure MTP speculative decoding correctly handles
|
||||
short prefill chunks that fall below the reorder_batch_threshold.
|
||||
"""
|
||||
short prefill chunks that fall below the reorder_batch_threshold."""
|
||||
# Set so large that both prefills will be classified as decodes in a mixed batch
|
||||
# note, with prefix caching we require chunk_size >= mamba_block_size
|
||||
chunk_size = 256 if not enable_prefix_caching else 2048
|
||||
|
||||
@@ -381,8 +381,7 @@ def test_pin_if_ready_none_when_absent():
|
||||
|
||||
def test_pin_if_ready_returns_unpinned_entry_when_not_ready():
|
||||
"""The producer needs "present but not ready" told apart from "absent":
|
||||
the first is NACKed as retryable, the second as a miss.
|
||||
"""
|
||||
the first is NACKed as retryable, the second as a miss."""
|
||||
cache = EmbeddingCache(num_blocks=4)
|
||||
cache.alloc("h", 1) # not-ready
|
||||
entry = cache.pin_if_ready("h")
|
||||
|
||||
@@ -131,8 +131,7 @@ def test_has_cache_item_false_when_not_consumer(monkeypatch):
|
||||
|
||||
def test_connector_keys_on_identifier_not_mm_hash(monkeypatch):
|
||||
"""The connector must key the encoder cache on feature.identifier (what
|
||||
has_cache_item is called with), NOT feature.mm_hash.
|
||||
"""
|
||||
has_cache_item is called with), NOT feature.mm_hash."""
|
||||
s = _make_scheduler(monkeypatch)
|
||||
req = _Request([_Feature("PROC_KEY", length=1, identifier="ENC_KEY")])
|
||||
s.update_state_after_alloc(req, 0)
|
||||
@@ -202,8 +201,7 @@ def test_loads_only_serves_hashes_touched_this_step(monkeypatch):
|
||||
|
||||
def test_repeated_reload_same_step_loads_once(monkeypatch):
|
||||
"""The same mm_hash requested twice in one step must appear in
|
||||
meta.loads exactly once.
|
||||
"""
|
||||
meta.loads exactly once."""
|
||||
s = _make_scheduler(monkeypatch)
|
||||
_seed_cached(s, "a", n_blocks=2)
|
||||
|
||||
@@ -226,8 +224,7 @@ def test_load_not_emitted_for_uncached_entry(monkeypatch):
|
||||
def test_load_pin_protects_blocks_until_completion(monkeypatch):
|
||||
"""Loaded blocks stay pinned until the worker reports the load memcpy
|
||||
complete. Observable: a save that needs eviction cannot reclaim a
|
||||
still-pinned loaded entry, but can once the completion report arrives.
|
||||
"""
|
||||
still-pinned loaded entry, but can once the completion report arrives."""
|
||||
# 2 blocks, both occupied by "a" (ready).
|
||||
s = _make_scheduler(monkeypatch, num_blocks=2)
|
||||
_seed_cached(s, "a", n_blocks=2)
|
||||
@@ -254,8 +251,7 @@ def test_load_pin_protects_blocks_until_completion(monkeypatch):
|
||||
|
||||
def _can_evict(s: ECCPUScheduler, mm_hash: str, n_blocks: int) -> bool:
|
||||
"""Whether a new save of `mm_hash` can claim `n_blocks`, i.e. whether the
|
||||
blocks currently held by other entries are reclaimable.
|
||||
"""
|
||||
blocks currently held by other entries are reclaimable."""
|
||||
s.update_state_after_alloc(_Request([_Feature(mm_hash, length=n_blocks)]), 0)
|
||||
meta = s.build_connector_meta(scheduler_output=None)
|
||||
return mm_hash in meta.saves
|
||||
@@ -315,8 +311,7 @@ def test_late_report_does_not_release_a_later_load_of_same_hash(monkeypatch):
|
||||
|
||||
def test_region_full_skips_save_and_never_blocks(monkeypatch):
|
||||
"""When the region is fully occupied by pinned entries, new saves are
|
||||
silently skipped and ensure_cache_available never blocks.
|
||||
"""
|
||||
silently skipped and ensure_cache_available never blocks."""
|
||||
s = _make_scheduler(monkeypatch, num_blocks=1)
|
||||
_seed_cached(s, "pinned", n_blocks=1)
|
||||
s._cache.pin("pinned")
|
||||
@@ -335,8 +330,7 @@ def test_region_full_skips_save_and_never_blocks(monkeypatch):
|
||||
|
||||
def test_producer_only_never_emits_loads(monkeypatch):
|
||||
"""A producer-only scheduler must never populate meta.loads, even when
|
||||
entries are ready.
|
||||
"""
|
||||
entries are ready."""
|
||||
s = _make_scheduler(monkeypatch, ec_role="ec_producer")
|
||||
req = _Request([_Feature("h1", length=1)])
|
||||
|
||||
@@ -378,8 +372,7 @@ def test_consumer_only_has_cache_item(monkeypatch):
|
||||
|
||||
def test_save_not_emitted_for_already_cached_entry(monkeypatch):
|
||||
"""An entry already in the cache (from a prior save) must not trigger
|
||||
a second allocation or appear in meta.saves again.
|
||||
"""
|
||||
a second allocation or appear in meta.saves again."""
|
||||
s = _make_scheduler(monkeypatch)
|
||||
req = _Request([_Feature("h1", length=1)])
|
||||
|
||||
@@ -400,8 +393,7 @@ def test_save_not_emitted_for_already_cached_entry(monkeypatch):
|
||||
|
||||
def test_update_connector_output_ignores_foreign_meta(monkeypatch):
|
||||
"""A worker output whose payload is not an ECCPUWorkerMetadata is a
|
||||
no-op — no cache mutation, no crash.
|
||||
"""
|
||||
no-op — no cache mutation, no crash."""
|
||||
s = _make_scheduler(monkeypatch)
|
||||
_seed_cached(s, "a", n_blocks=1)
|
||||
|
||||
@@ -415,8 +407,7 @@ def test_update_connector_output_ignores_foreign_meta(monkeypatch):
|
||||
|
||||
def test_update_connector_output_ignores_unknown_report(monkeypatch):
|
||||
"""Reports for a never-seen save hash or a transfer id this scheduler never
|
||||
dispatched are dropped rather than mutating unrelated state.
|
||||
"""
|
||||
dispatched are dropped rather than mutating unrelated state."""
|
||||
s = _make_scheduler(monkeypatch)
|
||||
s.update_connector_output(_WorkerOutput(saves=["gone"], loads=[4242]))
|
||||
assert s.has_cache_item("gone") is False
|
||||
@@ -428,8 +419,7 @@ def test_update_connector_output_ignores_unknown_report(monkeypatch):
|
||||
|
||||
def test_has_pending_push_work_tracks_inflight_save(monkeypatch):
|
||||
"""A dispatched-but-unconfirmed save keeps push work pending until the
|
||||
completion report marks it ready.
|
||||
"""
|
||||
completion report marks it ready."""
|
||||
s = _make_scheduler(monkeypatch)
|
||||
assert s.has_pending_push_work() is False
|
||||
|
||||
@@ -444,8 +434,7 @@ def test_has_pending_push_work_tracks_inflight_save(monkeypatch):
|
||||
|
||||
def test_has_pending_push_work_tracks_inflight_load(monkeypatch):
|
||||
"""A dispatched-but-unconfirmed load keeps push work pending until the
|
||||
unpin completion report arrives.
|
||||
"""
|
||||
unpin completion report arrives."""
|
||||
s = _make_scheduler(monkeypatch)
|
||||
_seed_cached(s, "a", n_blocks=1)
|
||||
assert s.has_pending_push_work() is False # ready + unpinned
|
||||
@@ -464,8 +453,7 @@ def test_has_pending_push_work_tracks_inflight_load(monkeypatch):
|
||||
|
||||
def test_shutdown_disables_roles_and_cleans_region(monkeypatch):
|
||||
"""After shutdown the scheduler serves no items and the region is
|
||||
cleaned up.
|
||||
"""
|
||||
cleaned up."""
|
||||
s = _make_scheduler(monkeypatch, num_blocks=2)
|
||||
_seed_cached(s, "a", n_blocks=2)
|
||||
assert s.has_cache_item("a") is True
|
||||
|
||||
@@ -98,8 +98,7 @@ def test_second_instance_opens_existing_file_and_shares_memory():
|
||||
def test_only_creator_unlinks_file_on_cleanup():
|
||||
"""Critical contract: if the non-creator unlinks, the creator's mmap path
|
||||
becomes a dangling backing file and a third opener would create a new one
|
||||
out from under the creator.
|
||||
"""
|
||||
out from under the creator."""
|
||||
instance_id = str(uuid.uuid4())
|
||||
r1 = ECSharedRegion(
|
||||
engine_id=instance_id,
|
||||
@@ -216,8 +215,7 @@ def test_madvise_einval_selects_fallback_for_whole_region(monkeypatch):
|
||||
|
||||
def test_non_creator_does_not_probe_or_populate(monkeypatch):
|
||||
"""Only the creator pre-faults. A second opener must not probe the advice
|
||||
nor re-touch pages the creator already populated.
|
||||
"""
|
||||
nor re-touch pages the creator already populated."""
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu import ec_shared_region as esr
|
||||
|
||||
instance_id = str(uuid.uuid4())
|
||||
@@ -314,8 +312,7 @@ def test_madvise_unexpected_oserror_propagates(monkeypatch):
|
||||
|
||||
def test_pin_memory_success_sets_flag(region):
|
||||
"""When cudaHostRegister returns 0, _is_pinned flips to True
|
||||
and cleanup will correspondingly call cudaHostUnregister.
|
||||
"""
|
||||
and cleanup will correspondingly call cudaHostUnregister."""
|
||||
fake_cudart = MagicMock()
|
||||
success = MagicMock()
|
||||
success.value = 0
|
||||
@@ -335,8 +332,7 @@ def test_pin_memory_success_sets_flag(region):
|
||||
|
||||
def test_pin_memory_failure_leaves_flag_false():
|
||||
"""If cudaHostRegister fails (non-zero), don't pretend it succeeded —
|
||||
cleanup must NOT call cudaHostUnregister on memory we never registered.
|
||||
"""
|
||||
cleanup must NOT call cudaHostUnregister on memory we never registered."""
|
||||
r = _make_region()
|
||||
try:
|
||||
fake_cudart = MagicMock()
|
||||
|
||||
@@ -363,8 +363,7 @@ def test_save_caches_coalesces_only_adjacent_blocks(
|
||||
|
||||
def test_save_caches_noop_when_mm_hash_not_in_saves(make_worker):
|
||||
"""When the scheduler hasn't pre-allocated blocks for ``mm_hash``,
|
||||
``save_caches`` + ``flush_saves`` is a pure no-op.
|
||||
"""
|
||||
``save_caches`` + ``flush_saves`` is a pure no-op."""
|
||||
worker = make_worker()
|
||||
sentinel = 0x42
|
||||
worker._region.blocks.fill_(sentinel)
|
||||
@@ -382,8 +381,7 @@ def test_save_caches_noop_when_mm_hash_not_in_saves(make_worker):
|
||||
)
|
||||
def test_save_caches_noop_for_non_save_rank(make_worker, tp_rank, pcp_rank):
|
||||
"""Only TP rank 0 + PCP rank 0 writes to mmap. All other combinations
|
||||
must skip the save entirely.
|
||||
"""
|
||||
must skip the save entirely."""
|
||||
worker = make_worker(tp_rank=tp_rank, pcp_rank=pcp_rank)
|
||||
sentinel = 0x42
|
||||
worker._region.blocks.fill_(sentinel)
|
||||
@@ -398,8 +396,7 @@ def test_save_caches_noop_for_non_save_rank(make_worker, tp_rank, pcp_rank):
|
||||
|
||||
def test_save_caches_raises_when_allocated_blocks_too_small(make_worker):
|
||||
"""``save_caches`` must raise ``AssertionError`` when the encoder output
|
||||
is larger than the allocated block space.
|
||||
"""
|
||||
is larger than the allocated block space."""
|
||||
worker = make_worker()
|
||||
src = torch.zeros(3 * _HIDDEN_DIM, dtype=_DTYPE)
|
||||
with pytest.raises(AssertionError, match="exceeds allocated blocks"):
|
||||
@@ -552,8 +549,7 @@ def test_start_load_caches_noop_when_loads_is_empty(make_worker):
|
||||
def test_start_load_caches_skips_cached_and_loads_new_in_same_step(make_worker):
|
||||
"""Every hash in ``meta.loads`` is copied from mmap, including one whose
|
||||
key is already resident in ``encoder_cache`` — there is no worker-side
|
||||
skip.
|
||||
"""
|
||||
skip."""
|
||||
worker = make_worker()
|
||||
n_blocks = 3
|
||||
src_orig = torch.arange(n_blocks * _HIDDEN_DIM, dtype=_DTYPE).reshape(
|
||||
@@ -782,8 +778,7 @@ def test_buffer_pool_is_reused_across_save_steps(make_worker):
|
||||
@_requires_accelerator
|
||||
def test_buffer_pool_is_reused_across_load_steps(make_worker):
|
||||
"""Once a load copy completes its descriptor buffers return to the pool and
|
||||
are reused by the next call — no reallocation.
|
||||
"""
|
||||
are reused by the next call — no reallocation."""
|
||||
worker = make_worker()
|
||||
worker._region.blocks[0].fill_(0x01)
|
||||
worker._region.blocks[1].fill_(0x02)
|
||||
@@ -854,8 +849,7 @@ def test_init_pins_memory_when_available(pin_available, expected_pinned):
|
||||
|
||||
def test_shutdown_calls_region_cleanup_and_swallows_errors(caplog_vllm):
|
||||
"""``shutdown`` must always call ``region.cleanup`` — and must never
|
||||
raise.
|
||||
"""
|
||||
raise."""
|
||||
worker = object.__new__(ECCPUWorker)
|
||||
mock_region = Mock(spec=ECSharedRegion)
|
||||
worker._region = mock_region
|
||||
|
||||
@@ -43,8 +43,7 @@ def assign(proxy):
|
||||
|
||||
def _drive(assign, e_urls, counts):
|
||||
"""Feed a sequence of request item-counts through the cursor, as
|
||||
fanout_encoder_primer does one request at a time.
|
||||
"""
|
||||
fanout_encoder_primer does one request at a time."""
|
||||
cursor = 0
|
||||
all_urls = []
|
||||
for count in counts:
|
||||
|
||||
@@ -40,8 +40,7 @@ def test_metadata_saves_dict_operations():
|
||||
|
||||
def test_metadata_loads_carry_a_transfer_id_and_blocks():
|
||||
"""Each load entry is a (transfer_id, block_ids) pair, so the worker can
|
||||
report the transfer rather than the mm_hash.
|
||||
"""
|
||||
report the transfer rather than the mm_hash."""
|
||||
meta = ECCPUConnectorMetadata()
|
||||
|
||||
meta.loads["mm_hash_1"] = (0, [5, 6, 7])
|
||||
|
||||
@@ -79,8 +79,7 @@ def test_request_finished_announces_not_ready_entry(monkeypatch):
|
||||
"""A save whose GPU->mmap copy hasn't been confirmed complete yet is
|
||||
still announced: the entry can't be evicted before it's ready, and a read
|
||||
arriving too early is NACKed NACK_NOT_READY for the consumer to retry —
|
||||
protecting it is not this method's job.
|
||||
"""
|
||||
protecting it is not this method's job."""
|
||||
s = _sched_gate_off(monkeypatch)
|
||||
s._nixl_enabled = True
|
||||
s._peer_host, s._peer_port = "1.2.3.4", 5601
|
||||
|
||||
@@ -129,8 +129,7 @@ def test_producer_session_grant_nacks_missing_entry():
|
||||
|
||||
def test_producer_session_grant_nacks_not_ready_entry_as_retryable():
|
||||
"""An announced entry whose save is still in flight is distinguished from a
|
||||
miss, so the consumer retries instead of falling back to local compute.
|
||||
"""
|
||||
miss, so the consumer retries instead of falling back to local compute."""
|
||||
cache = EmbeddingCache(num_blocks=8)
|
||||
cache.alloc("h1", 2) # not ready
|
||||
s = _make_producer_session(cache)
|
||||
@@ -438,8 +437,7 @@ def test_consumer_session_poll_nack_goes_to_tombstoned():
|
||||
|
||||
def test_consumer_session_poll_not_ready_nack_goes_to_retryable():
|
||||
"""A not-ready NACK must not tombstone: tombstoning admits the request with
|
||||
no embedding, which is fatal once the media has been rewritten away.
|
||||
"""
|
||||
no embedding, which is fatal once the media has been rewritten away."""
|
||||
import msgspec
|
||||
|
||||
s = _make_consumer_session()
|
||||
|
||||
@@ -68,8 +68,7 @@ class _StubEngineCoreActor(EngineCoreActorMixin):
|
||||
# in ``tests/entrypoints/test_api_server_process_manager.py``.
|
||||
def _bind_and_report_worker(listen_address, sock, args, client_config):
|
||||
"""Bind ROUTER/PULL with a kernel-assigned port, report the actual
|
||||
endpoints back via ``actual_address_pipe``, then exit.
|
||||
"""
|
||||
endpoints back via ``actual_address_pipe``, then exit."""
|
||||
ctx = zmq.Context()
|
||||
try:
|
||||
in_sock = make_zmq_socket(
|
||||
|
||||
@@ -607,8 +607,7 @@ def _cadenced_dp_engine_core(
|
||||
monkeypatch, results: list[tuple[bool, bool]], dp_sync_interval: int = 32
|
||||
):
|
||||
"""A bare DPEngineCoreProc whose all-reduce is scripted by `results`;
|
||||
returns the core and the step numbers at which the all-reduce ran.
|
||||
"""
|
||||
returns the core and the step numbers at which the all-reduce ran."""
|
||||
core = object.__new__(DPEngineCoreProc)
|
||||
core.dp_group = object()
|
||||
core.dp_sync_interval = dp_sync_interval
|
||||
@@ -627,8 +626,7 @@ def _cadenced_dp_engine_core(
|
||||
|
||||
def test_dp_sync_interval_default_is_16():
|
||||
"""Regression pin: lowering the default narrows the mid-wave pause tail
|
||||
for async RL, where the engine is rarely idle when pause lands.
|
||||
"""
|
||||
for async RL, where the engine is rarely idle when pause lands."""
|
||||
assert ParallelConfig.dp_sync_interval == 16
|
||||
|
||||
|
||||
@@ -646,8 +644,7 @@ def test_dp_sync_interval_normal_wave(monkeypatch, dp_sync_interval: int):
|
||||
def test_dp_sync_interval_idle_pause_consensus_on_first_step(monkeypatch):
|
||||
"""A pause of an idle engine arms every rank before its kick-started
|
||||
first step, so the step-1 sync reaches consensus after one dummy batch
|
||||
regardless of the configured cadence.
|
||||
"""
|
||||
regardless of the configured cadence."""
|
||||
core, synced = _cadenced_dp_engine_core(
|
||||
monkeypatch, [(False, True)], dp_sync_interval=32
|
||||
)
|
||||
@@ -673,8 +670,7 @@ def _pausable_engine_core_proc() -> EngineCoreProc:
|
||||
@pytest.mark.parametrize("deferred", [False, True])
|
||||
def test_pause_synchronizes_device_before_cache_reset(deferred: bool):
|
||||
"""A resolved pause promises an idle device: the barrier must run before
|
||||
caches are cleared and before the caller is unblocked.
|
||||
"""
|
||||
caches are cleared and before the caller is unblocked."""
|
||||
core = _pausable_engine_core_proc()
|
||||
core.engines_running = deferred
|
||||
order: list[str] = []
|
||||
|
||||
@@ -249,8 +249,7 @@ def test_dplb_non_late_interaction_still_uses_lb():
|
||||
|
||||
def test_dplb_burst_round_robins_despite_snapshot_rebinds():
|
||||
"""A stats snapshot rebind wipes the optimistic lb_engines increments;
|
||||
the exact in-flight floor must keep a burst spreading round-robin.
|
||||
"""
|
||||
the exact in-flight floor must keep a burst spreading round-robin."""
|
||||
client = _make_dplb_client(num_engines=4)
|
||||
|
||||
for _ in range(4):
|
||||
@@ -325,8 +324,7 @@ async def test_dplb_scale_down_routes_after_stale_stats_snapshot():
|
||||
|
||||
def test_dplb_snapshot_backpressure_overrides_inflight():
|
||||
"""An engine reported heavily loaded by the coordinator is avoided even
|
||||
when this client has routed nothing to it.
|
||||
"""
|
||||
when this client has routed nothing to it."""
|
||||
client = _make_dplb_client(num_engines=2)
|
||||
client.lb_engines = [[5, 10, 0.0], [0, 0, 0.0]]
|
||||
|
||||
@@ -339,8 +337,7 @@ def test_dplb_snapshot_backpressure_overrides_inflight():
|
||||
|
||||
def test_dplb_kv_pressure_amplifies_waiting_penalty():
|
||||
"""A waiting queue on a KV-bound engine (slow drain) is penalized, while
|
||||
the same queue with low KV usage is not (e.g. transient burst).
|
||||
"""
|
||||
the same queue with low KV usage is not (e.g. transient burst)."""
|
||||
client = _make_dplb_client(num_engines=2)
|
||||
# Engine 0 has a smaller total but is KV-bound with a queue.
|
||||
client.lb_engines = [[5, 10, 1.0], [0, 20, 0.2]]
|
||||
|
||||
@@ -30,8 +30,7 @@ def _make_processor(num_logprobs: int) -> LogprobsProcessor:
|
||||
|
||||
def test_drops_trailing_sentinel_columns():
|
||||
"""A request that asked for 3 custom token logprobs but ended up in a
|
||||
batch padded to width 5 must not surface the trailing -inf entries.
|
||||
"""
|
||||
batch padded to width 5 must not surface the trailing -inf entries."""
|
||||
processor = _make_processor(num_logprobs=3)
|
||||
|
||||
sampled = 42
|
||||
|
||||
@@ -229,8 +229,7 @@ def test_request_stream_interval_raises_but_not_below_engine_default(
|
||||
):
|
||||
"""A per-request stream_interval can raise the interval above the engine
|
||||
default but not below it (values under the default clamp up), without
|
||||
altering the generated text.
|
||||
"""
|
||||
altering the generated text."""
|
||||
engine_stream_interval = 5
|
||||
# Request 0 (below the default) clamps up to 5; request 1 raises it to 10.
|
||||
request_stream_intervals = [1, 10]
|
||||
|
||||
@@ -80,8 +80,7 @@ def _split_hnd_cache(spec) -> torch.Tensor:
|
||||
|
||||
def _packed_nhd_cache(spec) -> torch.Tensor:
|
||||
"""Logical (num_blocks, heads, block_size, 2 * head_size) over an NHD
|
||||
physical layout — the FlashAttention/FlashInfer/Triton/Flex form.
|
||||
"""
|
||||
physical layout — the FlashAttention/FlashInfer/Triton/Flex form."""
|
||||
return torch.zeros(
|
||||
NUM_BLOCKS,
|
||||
spec.block_size,
|
||||
|
||||
@@ -863,8 +863,7 @@ def test_parallelism_agnostic_excluded(kv_cache_groups: list[KVCacheGroupSpec]):
|
||||
)
|
||||
def test_canonical_layout_gate(kv_cache_groups, certified):
|
||||
"""The canonical layout certifies portability group by group; none of
|
||||
these shapes are portable in the direct layout.
|
||||
"""
|
||||
these shapes are portable in the direct layout."""
|
||||
assert not _parallelism_agnostic(kv_cache_groups)
|
||||
assert _parallelism_agnostic(kv_cache_groups, canonical=True) is certified
|
||||
|
||||
@@ -872,8 +871,7 @@ def test_canonical_layout_gate(kv_cache_groups, certified):
|
||||
def test_canonical_layout_certifies_v2_model_runner():
|
||||
"""Canonical bytes are certified per layer against live tensor strides at
|
||||
registration, so the static gate must not depend on the model-runner
|
||||
version — the v2 runner is the case the canonical layout exists for.
|
||||
"""
|
||||
version — the v2 runner is the case the canonical layout exists for."""
|
||||
groups = _groups(_full_attention_spec())
|
||||
assert _parallelism_agnostic(groups, canonical=True, v2=True)
|
||||
|
||||
|
||||
@@ -117,8 +117,7 @@ def _spec_cls_with_metric_definitions(
|
||||
metric_definitions: dict[str, Any],
|
||||
) -> type:
|
||||
"""Build a fake offloading spec class reporting the given metric
|
||||
definitions, so tests don't need to patch the real CPU spec.
|
||||
"""
|
||||
definitions, so tests don't need to patch the real CPU spec."""
|
||||
|
||||
class _FakeOffloadingSpec:
|
||||
@staticmethod
|
||||
@@ -188,8 +187,7 @@ def test_build_kv_connector_stats_with_empty_dict():
|
||||
|
||||
def test_build_kv_connector_stats_reconstructs_offload_stats():
|
||||
"""Test that OffloadingConnector stats are properly reconstructed with
|
||||
correct data.
|
||||
"""
|
||||
correct data."""
|
||||
serialized_data = {
|
||||
_StatsKey.TYPES: {
|
||||
LOAD_BYTES: _MetricType.COUNTER,
|
||||
|
||||
@@ -1583,8 +1583,7 @@ _SCAN_BEHAVIOR = {
|
||||
@pytest.mark.parametrize("result", list(LookupResult))
|
||||
def test_scan_behavior_declared_for_every_lookup_result(result: LookupResult):
|
||||
"""Whether a result keeps a sliding-window streak alive decides how wide
|
||||
the demanded chunk set gets, so every member needs deliberate behavior.
|
||||
"""
|
||||
the demanded chunk set gets, so every member needs deliberate behavior."""
|
||||
assert result in _SCAN_BEHAVIOR, f"{result} has no declared scan behavior"
|
||||
expected_lookups, expected_end = _SCAN_BEHAVIOR[result]
|
||||
|
||||
@@ -2035,8 +2034,7 @@ def test_request_level_policy_stores_all_blocks(request_runner, async_scheduling
|
||||
|
||||
def test_loads_do_not_populate_fence_index(request_runner):
|
||||
"""Loads don't populate _block_id_to_pending_jobs (protected by
|
||||
delay_free_blocks while in flight).
|
||||
"""
|
||||
delay_free_blocks while in flight)."""
|
||||
runner = request_runner(
|
||||
blocks_per_chunk=3,
|
||||
block_size=4,
|
||||
@@ -2106,8 +2104,7 @@ def test_fence_at_update_state_after_alloc(request_runner):
|
||||
def test_fence_at_build_store_jobs(request_runner):
|
||||
"""A new prefill (no load -> update_state_after_alloc returns early)
|
||||
reusing a finished request's pending-store block is flushed by
|
||||
_build_store_jobs's fence.
|
||||
"""
|
||||
_build_store_jobs's fence."""
|
||||
runner = request_runner(
|
||||
blocks_per_chunk=1,
|
||||
block_size=4,
|
||||
@@ -2157,8 +2154,7 @@ def test_fence_at_build_store_jobs(request_runner):
|
||||
@pytest.mark.parametrize("async_scheduling", [True, False])
|
||||
def test_complete_store_called_per_job(request_runner, async_scheduling: bool):
|
||||
"""complete_store fires per-job, not deferred to request finish.
|
||||
Each call carries only that store's keys.
|
||||
"""
|
||||
Each call carries only that store's keys."""
|
||||
tokens_per_block = 4
|
||||
blocks_per_chunk = 3
|
||||
tokens_per_chunk = tokens_per_block * blocks_per_chunk
|
||||
@@ -2387,8 +2383,7 @@ def test_offload_prompt_only(request_runner, async_scheduling: bool):
|
||||
@pytest.mark.parametrize("async_scheduling", [True, False])
|
||||
def test_reset_cache(request_runner, async_scheduling: bool):
|
||||
"""reset_cache flushes in-flight loads, calls manager.reset_cache(), resets
|
||||
next_stored_chunk_idx for active requests and clears job tracking.
|
||||
"""
|
||||
next_stored_chunk_idx for active requests and clears job tracking."""
|
||||
block_size = 4
|
||||
blocks_per_chunk = 3
|
||||
tokens_per_chunk = block_size * blocks_per_chunk
|
||||
@@ -2934,8 +2929,7 @@ def test_stale_sliding_window_block_after_prepare_store_failure(
|
||||
@pytest.mark.parametrize("async_scheduling", [True, False])
|
||||
def test_skip_reading_prefix_cache(request_runner, async_scheduling: bool):
|
||||
"""When skip_reading_prefix_cache=True, the offloading connector must not
|
||||
load any blocks from CPU even if a matching prefix is cached there.
|
||||
"""
|
||||
load any blocks from CPU even if a matching prefix is cached there."""
|
||||
block_size = 4
|
||||
blocks_per_chunk = 3
|
||||
tokens_per_chunk = block_size * blocks_per_chunk
|
||||
@@ -3029,8 +3023,7 @@ def test_max_load_tokens_limits_external_load(request_runner, async_scheduling:
|
||||
|
||||
class TestEagle:
|
||||
"""Tests for Eagle/MTP speculative decoding support in the offloading
|
||||
connector scheduler — both _lookup() unit tests and integration tests.
|
||||
"""
|
||||
connector scheduler — both _lookup() unit tests and integration tests."""
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# Helpers
|
||||
@@ -3572,8 +3565,7 @@ class TestEagle:
|
||||
self, request_runner, async_scheduling: bool
|
||||
):
|
||||
"""Eagle sliding-window group stores all prompt blocks but excludes
|
||||
the trailing chunk while decoding.
|
||||
"""
|
||||
the trailing chunk while decoding."""
|
||||
block_size = 4
|
||||
sliding_window = 8
|
||||
num_gpu_blocks = 100
|
||||
@@ -3625,8 +3617,7 @@ class TestEagle:
|
||||
self, request_runner, async_scheduling: bool
|
||||
):
|
||||
"""An eagle group with a single-block prompt stores it at the end of
|
||||
prefill: prompt blocks are stable, so no tail is held back.
|
||||
"""
|
||||
prefill: prompt blocks are stable, so no tail is held back."""
|
||||
block_size = 4
|
||||
blocks_per_chunk = 1
|
||||
tokens_per_chunk = block_size * blocks_per_chunk
|
||||
@@ -4263,8 +4254,7 @@ def test_request_finished_mixed_full_attn_and_sliding_window(
|
||||
|
||||
def test_chunked_local_attention_reports_its_chunk_window():
|
||||
"""Llama 4 uses chunked local attention, which used to trip the
|
||||
FullAttentionSpec assert and kill the engine at startup.
|
||||
"""
|
||||
FullAttentionSpec assert and kill the engine at startup."""
|
||||
spec = ChunkedLocalAttentionSpec(
|
||||
block_size=16,
|
||||
num_kv_heads=2,
|
||||
@@ -4503,8 +4493,7 @@ def test_retention_interval_zero_stores_only_replay_boundary(
|
||||
def _shared_kv_mtp_config():
|
||||
"""Speculative config for a shared-group MTP model: eagle-family method
|
||||
whose drafter layer merges into a target KV-cache group, so no group
|
||||
self-identifies as a drafter group (issue #52735).
|
||||
"""
|
||||
self-identifies as a drafter group (issue #52735)."""
|
||||
spec = MagicMock(name="shared_kv_mtp_spec")
|
||||
spec.use_eagle.return_value = True
|
||||
spec.use_eagle_block_drop.return_value = True
|
||||
@@ -4518,14 +4507,12 @@ def _shared_kv_mtp_config():
|
||||
class TestSharedGroupMTPOffload:
|
||||
"""Regression tests for issue #52735: OffloadingConnector must keep
|
||||
serving when speculative decoding is enabled but no KV-cache group is
|
||||
annotated as a drafter group (shared-group MTP models).
|
||||
"""
|
||||
annotated as a drafter group (shared-group MTP models)."""
|
||||
|
||||
def test_no_annotation_marks_no_groups(self, request_runner):
|
||||
"""Spec decode on + zero annotated groups must NOT mark every group
|
||||
as a drafter group; the full store->load roundtrip must match the
|
||||
non-speculative behavior of test_two_groups_full_and_sliding_window.
|
||||
"""
|
||||
non-speculative behavior of test_two_groups_full_and_sliding_window."""
|
||||
block_size = 4
|
||||
kv_cache_groups = [
|
||||
KVCacheGroupSpec(
|
||||
@@ -4582,8 +4569,7 @@ class TestSharedGroupMTPOffload:
|
||||
class TestMambaHybridOffloadServing:
|
||||
"""Store->finish->lookup flows on a full-attention + mamba-align hybrid,
|
||||
with a manager that only HITs keys that were actually stored. Guards the
|
||||
two collapse routes of issue #52735.
|
||||
"""
|
||||
two collapse routes of issue #52735."""
|
||||
|
||||
BLOCK = 4
|
||||
MAMBA_BLOCK = 16
|
||||
@@ -4712,8 +4698,7 @@ class TestMambaHybridOffloadServing:
|
||||
"""Route 1 of #52735: with spec decode on and no drafter annotation,
|
||||
serving must equal the non-speculative baseline (was 0 before the
|
||||
fix: the all-groups fallback marked the mamba group as a drafter and
|
||||
the volatile-tail pop consumed its only servable chunk).
|
||||
"""
|
||||
the volatile-tail pop consumed its only servable chunk)."""
|
||||
baseline = self._roundtrip_served_tokens(self._make_scheduler(None))
|
||||
assert baseline == 16
|
||||
served = self._roundtrip_served_tokens(
|
||||
@@ -4726,8 +4711,7 @@ class TestMambaHybridOffloadServing:
|
||||
group must not drag the confirmed boundary below a coarser sibling's
|
||||
chunk granularity. The widened query makes the volatile-tail pop
|
||||
land on an extra queried chunk, holding the boundary at 16 tokens
|
||||
(was 0 before the fix: 4 hits -> pop -> 12 tokens < mamba chunk).
|
||||
"""
|
||||
(was 0 before the fix: 4 hits -> pop -> 12 tokens < mamba chunk)."""
|
||||
scheduler = self._make_scheduler(None, mamba_eagle=True)
|
||||
assert [c.is_eagle_group for c in scheduler.config.kv_group_configs] == [
|
||||
True,
|
||||
|
||||
@@ -80,8 +80,7 @@ def _allocate_kv_caches(
|
||||
|
||||
def _single_rank_vllm_config(total_kv_heads: int):
|
||||
"""A one-rank (TP=1) parallel config, as canonical mappings are derived
|
||||
from it.
|
||||
"""
|
||||
from it."""
|
||||
vllm_config = MagicMock()
|
||||
parallel_config = vllm_config.parallel_config
|
||||
parallel_config.tensor_parallel_size = 1
|
||||
|
||||
@@ -148,8 +148,7 @@ def test_multiturn_lifecycle():
|
||||
(do_remote_decode=True). Finishes LENGTH_CAPPED with remote_block_ids.
|
||||
Turn 2: P receives remote_block_ids from D. P pulls KV from D because
|
||||
remote_block_ids is not None and external tokens > 0. Computes only
|
||||
new tokens, finishes LENGTH_CAPPED.
|
||||
"""
|
||||
new tokens, finishes LENGTH_CAPPED."""
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG,
|
||||
)
|
||||
@@ -195,8 +194,7 @@ def test_multiturn_lifecycle():
|
||||
|
||||
def test_first_turn_no_remote_blocks():
|
||||
"""First turn: P has no remote_block_ids from D yet.
|
||||
Standard local prefill, returns kv_transfer_params for future turns.
|
||||
"""
|
||||
Standard local prefill, returns kv_transfer_params for future turns."""
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG,
|
||||
)
|
||||
@@ -276,8 +274,7 @@ def test_abort_p_side_non_length_capped():
|
||||
|
||||
def test_remote_blocks_exceed_prompt_tokens():
|
||||
"""D provides more remote tokens than P's prompt needs.
|
||||
P caps external tokens to prompt length.
|
||||
"""
|
||||
P caps external tokens to prompt length."""
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG,
|
||||
)
|
||||
@@ -313,8 +310,7 @@ def test_remote_blocks_exceed_prompt_tokens():
|
||||
def test_p_node_pulls_partial_last_block_from_d():
|
||||
"""D sends remote_block_ids with partially filled last block.
|
||||
remote_num_tokens < len(remote_block_ids) * block_size.
|
||||
P pulls only remote_num_tokens worth of external tokens.
|
||||
"""
|
||||
P pulls only remote_num_tokens worth of external tokens."""
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG,
|
||||
)
|
||||
@@ -359,8 +355,7 @@ def test_p_node_pulls_partial_last_block_from_d():
|
||||
|
||||
def test_add_new_req_to_recv_populates_remote_meta():
|
||||
"""add_new_req_to_recv correctly populates RemoteMeta for P-node
|
||||
bi-directional KV pull from D.
|
||||
"""
|
||||
bi-directional KV pull from D."""
|
||||
meta = NixlConnectorMetadata()
|
||||
kv_params = {
|
||||
"remote_block_ids": [[0, 1, 2]],
|
||||
@@ -388,8 +383,7 @@ def test_add_new_req_to_recv_populates_remote_meta():
|
||||
|
||||
def test_build_connector_meta_recv_entries():
|
||||
"""P-node scheduler: do_remote_decode=True + remote_block_ids →
|
||||
_reqs_need_recv populated, build_connector_meta produces reqs_to_recv.
|
||||
"""
|
||||
_reqs_need_recv populated, build_connector_meta produces reqs_to_recv."""
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG,
|
||||
)
|
||||
@@ -463,8 +457,7 @@ def test_p_node_pull_kv_from_d(dist_init):
|
||||
)
|
||||
def test_p_node_pull_then_send_kv(dist_init):
|
||||
"""Full P-node bi-directional: pull KV from D → prefill →
|
||||
send KV back to D via notification.
|
||||
"""
|
||||
send KV back to D via notification."""
|
||||
connector, worker = _make_connector_with_fake_worker()
|
||||
meta = _make_p_node_recv_metadata("req-p2", [10, 11], [20, 21])
|
||||
_do_load_kv(connector, meta)
|
||||
@@ -511,8 +504,7 @@ def test_p_node_deferred_pull_on_no_handshake(dist_init):
|
||||
def test_d_node_request_finished_returns_kv_params():
|
||||
"""D-node request_finished returns kv_transfer_params with
|
||||
do_remote_decode=True, remote_block_ids, remote_num_tokens
|
||||
for P to pull. These params go directly to P node.
|
||||
"""
|
||||
for P to pull. These params go directly to P node."""
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG,
|
||||
)
|
||||
@@ -593,8 +585,7 @@ def test_d_node_request_finished_remote_num_tokens():
|
||||
|
||||
def test_d_node_partial_last_block_remote_num_tokens():
|
||||
"""D-node: remote_num_tokens < len(remote_block_ids) * block_size
|
||||
when last block is partially filled.
|
||||
"""
|
||||
when last block is partially filled."""
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG,
|
||||
)
|
||||
@@ -628,8 +619,7 @@ def test_no_double_read_blocks_after_reschedule():
|
||||
bidirectional request (once on initial schedule, once after
|
||||
WAITING_FOR_REMOTE_KVS → reschedule). The _remote_blocks_processed
|
||||
flag must prevent the request from being added to _reqs_need_recv
|
||||
twice, which would cause P to read D's blocks twice.
|
||||
"""
|
||||
twice, which would cause P to read D's blocks twice."""
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG,
|
||||
)
|
||||
@@ -682,8 +672,7 @@ def test_remote_num_tokens_bounded_by_blocks():
|
||||
"""Edge case 2: D-node request_finished must return
|
||||
remote_num_tokens <= len(remote_block_ids) * block_size.
|
||||
request.num_tokens includes the last sampled token which has no KV
|
||||
in the cache, so remote_num_tokens must use num_computed_tokens.
|
||||
"""
|
||||
in the cache, so remote_num_tokens must use num_computed_tokens."""
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG,
|
||||
)
|
||||
@@ -719,8 +708,7 @@ def test_remote_num_tokens_bounded_by_blocks():
|
||||
def test_kv_recompute_threshold_skips_small_transfer():
|
||||
"""Edge case 3: When remote tokens are below kv_recompute_threshold,
|
||||
P should skip the remote pull and compute locally instead of
|
||||
entering WAITING_FOR_REMOTE_KVS.
|
||||
"""
|
||||
entering WAITING_FOR_REMOTE_KVS."""
|
||||
threshold = 256
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector_extra_config={
|
||||
@@ -762,8 +750,7 @@ def test_p_node_finished_holds_blocks_for_d():
|
||||
"""Edge case 4: P-node finishes with FINISHED_LENGTH_CAPPED and
|
||||
do_remote_decode=True. P must hold blocks (delay_free=True) and
|
||||
return kv_transfer_params with do_remote_prefill=True so D can
|
||||
read P's blocks.
|
||||
"""
|
||||
read P's blocks."""
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG,
|
||||
)
|
||||
@@ -804,8 +791,7 @@ def test_p_node_finished_holds_blocks_for_d():
|
||||
def test_cache_miss_first_turn_no_remote_pull():
|
||||
"""Edge case 5: First turn with do_remote_decode=True but no
|
||||
remote_block_ids (cache MISS). P should prefill locally with
|
||||
num_external_tokens=0 and not enter WAITING_FOR_REMOTE_KVS.
|
||||
"""
|
||||
num_external_tokens=0 and not enter WAITING_FOR_REMOTE_KVS."""
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG,
|
||||
)
|
||||
@@ -840,8 +826,7 @@ def test_cache_miss_first_turn_no_remote_pull():
|
||||
def test_partial_remote_tokens_less_than_prompt():
|
||||
"""Edge case 6: D's remote_num_tokens covers only part of P's
|
||||
prompt. P should pull remote_num_tokens worth of external tokens
|
||||
and compute the rest locally.
|
||||
"""
|
||||
and compute the rest locally."""
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG,
|
||||
)
|
||||
@@ -886,8 +871,7 @@ def test_partial_remote_tokens_less_than_prompt():
|
||||
def test_remote_blocks_processed_flag_persists():
|
||||
"""Edge case 7: After recv completes and request is rescheduled,
|
||||
the _remote_blocks_processed flag in kv_transfer_params prevents
|
||||
the bidirectional path from re-entering _reqs_need_recv.
|
||||
"""
|
||||
the bidirectional path from re-entering _reqs_need_recv."""
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG,
|
||||
)
|
||||
@@ -959,8 +943,7 @@ _REMOTE = FakeNixlConnectorWorker.REMOTE_ENGINE_ID
|
||||
)
|
||||
def test_turn2_deadline_gate(dist_init, offset, expiry_delta, expect_declined):
|
||||
"""P declines on the turn-2 readback if the deadline is known,
|
||||
near-expiry and a handshake clock offset is known
|
||||
"""
|
||||
near-expiry and a handshake clock offset is known"""
|
||||
connector, worker = _make_connector_with_fake_worker()
|
||||
worker._engine_clock_offset[_REMOTE] = offset
|
||||
expiry_time = None if expiry_delta is None else time.perf_counter() + expiry_delta
|
||||
@@ -996,8 +979,7 @@ def test_turn2_deadline_gate(dist_init, offset, expiry_delta, expect_declined):
|
||||
)
|
||||
def test_turn2_full_prefix_hit_with_expired_deadline_skips_gate(dist_init):
|
||||
"""A full local prefix hit issues no READ. The expiry gate must be skipped
|
||||
even when D's deadline is expired.
|
||||
"""
|
||||
even when D's deadline is expired."""
|
||||
connector, worker = _make_connector_with_fake_worker()
|
||||
worker._engine_clock_offset[_REMOTE] = 0.0
|
||||
|
||||
@@ -1050,8 +1032,7 @@ def test_d_node_request_finished_exports_blocks_expiry_time():
|
||||
|
||||
def test_handshake_listener_appends_perf_counter_frame():
|
||||
"""The handshake reply carries a 2nd frame with the listener's live
|
||||
perf_counter.
|
||||
"""
|
||||
perf_counter."""
|
||||
import threading
|
||||
|
||||
import msgspec
|
||||
|
||||
@@ -19,15 +19,13 @@ class _StubLMCacheMPConnector:
|
||||
test only asserts on the connector *name* and the ``extra_config`` dict
|
||||
produced by ``VllmConfig``, never instantiates the connector, so a bare
|
||||
placeholder class is sufficient. Not subclassing ``SupportsHMA`` mirrors
|
||||
the real connector's HMA support (it does not support HMA either).
|
||||
"""
|
||||
the real connector's HMA support (it does not support HMA either)."""
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def stub_lmcache_mp_connector(monkeypatch):
|
||||
"""Replace the lazy loader so VllmConfig.__post_init__ does not import
|
||||
``lmcache_mp_connector`` (and thus ``lmcache``) during config tests.
|
||||
"""
|
||||
``lmcache_mp_connector`` (and thus ``lmcache``) during config tests."""
|
||||
monkeypatch.setitem(
|
||||
KVConnectorFactory._registry,
|
||||
"LMCacheMPConnector",
|
||||
@@ -107,8 +105,7 @@ def _build_config(
|
||||
enable_cumem_allocator: bool = False,
|
||||
) -> VllmConfig:
|
||||
"""Build a VllmConfig that exercises _verify_kv_transfer_compat without
|
||||
requiring a real model (avoids HF downloads in CI).
|
||||
"""
|
||||
requiring a real model (avoids HF downloads in CI)."""
|
||||
from types import SimpleNamespace
|
||||
|
||||
kv_transfer_config = (
|
||||
@@ -134,8 +131,7 @@ def test_kv_connector_rejects_expandable_segments(monkeypatch, kv_connector):
|
||||
invalidated when expandable_segments lets the CUDA VMM allocator remap
|
||||
the underlying physical pages. We can't enumerate every connector that
|
||||
does this (especially OOT ones), so reject the combination whenever any
|
||||
connector is configured.
|
||||
"""
|
||||
connector is configured."""
|
||||
monkeypatch.setenv("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
|
||||
with pytest.raises(ValueError, match="expandable_segments"):
|
||||
_build_config(kv_connector=kv_connector)
|
||||
@@ -143,8 +139,7 @@ def test_kv_connector_rejects_expandable_segments(monkeypatch, kv_connector):
|
||||
|
||||
def test_kv_connector_allows_expandable_segments_with_sleep_mode(monkeypatch):
|
||||
"""Sleep mode routes KV allocations through CuMemAllocator's pool, which
|
||||
auto-disables expandable_segments (see #40812).
|
||||
"""
|
||||
auto-disables expandable_segments (see #40812)."""
|
||||
monkeypatch.setenv("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
|
||||
_build_config(kv_connector="NixlConnector", enable_sleep_mode=True)
|
||||
|
||||
@@ -167,8 +162,7 @@ def test_kv_connector_allows_other_alloc_conf(monkeypatch):
|
||||
|
||||
def test_no_kv_connector_ignores_expandable_segments(monkeypatch):
|
||||
"""The expandable_segments check only applies when a KV connector is
|
||||
configured.
|
||||
"""
|
||||
configured."""
|
||||
monkeypatch.setenv("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
|
||||
_build_config(kv_connector=None)
|
||||
|
||||
|
||||
@@ -50,8 +50,7 @@ def _make_flexkv_module(
|
||||
impl_mock: MagicMock,
|
||||
) -> tuple[types.ModuleType, types.ModuleType]:
|
||||
"""Build a fake ``flexkv`` package hierarchy that returns *impl_mock*
|
||||
when ``FlexKVConnectorV1Impl`` is instantiated.
|
||||
"""
|
||||
when ``FlexKVConnectorV1Impl`` is instantiated."""
|
||||
flexkv_mod = types.ModuleType("flexkv")
|
||||
integration_mod = types.ModuleType("flexkv.integration")
|
||||
vllm_mod = types.ModuleType("flexkv.integration.vllm")
|
||||
@@ -72,8 +71,7 @@ def _make_flexkv_module(
|
||||
|
||||
def _install_flexkv_mock(impl_mock: MagicMock):
|
||||
"""Insert fake flexkv modules into sys.modules and return a context that
|
||||
cleans them up afterwards.
|
||||
"""
|
||||
cleans them up afterwards."""
|
||||
flexkv_mod, adapter_mod = _make_flexkv_module(impl_mock)
|
||||
mods = {
|
||||
"flexkv": flexkv_mod,
|
||||
@@ -106,8 +104,7 @@ def _build_connector(vllm_config: VllmConfig, impl_mock: MagicMock):
|
||||
|
||||
class TestFlexKVConnectorImportError:
|
||||
"""FlexKVConnectorV1 should fail with a helpful message when flexkv is
|
||||
absent.
|
||||
"""
|
||||
absent."""
|
||||
|
||||
def test_import_error_message(self):
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.flexkv_connector import (
|
||||
|
||||
@@ -176,8 +176,7 @@ def test_engine_unwraps_handshake_metadata_for_legacy_connector(
|
||||
) -> None:
|
||||
"""Engine core always asks workers for `(pp_rank, tp_rank)`-keyed metadata,
|
||||
then unwraps to `{tp_rank: metadata}` for a connector that has not opted
|
||||
into PP-aware handshake (single-PP producer, all `pp_rank == 0`).
|
||||
"""
|
||||
into PP-aware handshake (single-PP producer, all `pp_rank == 0`)."""
|
||||
metadata_0 = _Metadata()
|
||||
metadata_1 = _Metadata()
|
||||
connector = _LegacyConnector()
|
||||
@@ -200,8 +199,7 @@ def test_engine_rejects_pp_producer_for_legacy_connector(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""A connector that has not opted into PP-aware handshake must not silently
|
||||
drop metadata from `pp_rank > 0`; engine core init raises instead.
|
||||
"""
|
||||
drop metadata from `pp_rank > 0`; engine core init raises instead."""
|
||||
connector = _LegacyConnector()
|
||||
|
||||
with pytest.raises(ValueError, match="does not support PP-disaggregated"):
|
||||
@@ -216,8 +214,7 @@ def test_engine_passes_handshake_metadata_through_for_pp_aware_connector(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""A PP-aware connector receives the full `(pp_rank, tp_rank)`-keyed dict
|
||||
unchanged.
|
||||
"""
|
||||
unchanged."""
|
||||
metadata_0 = _Metadata()
|
||||
metadata_1 = _Metadata()
|
||||
connector = _PPAwareConnector()
|
||||
|
||||
@@ -131,8 +131,7 @@ class TestHf3fsClientResourceManagement:
|
||||
|
||||
def test_close_idempotent_and_handles_cleared(self, tmp_path):
|
||||
"""Multiple close() calls must not raise; deregister_fd called exactly
|
||||
once, all handles set to None, shm.close() invoked.
|
||||
"""
|
||||
once, all handles set to None, shm.close() invoked."""
|
||||
client, shm_r, shm_w = self._make_client(tmp_path)
|
||||
|
||||
with (
|
||||
@@ -238,8 +237,7 @@ class TestHf3fsClientResourceManagement:
|
||||
|
||||
def test_constructor_failure_does_not_close_unallocated_shm(self, tmp_path):
|
||||
"""Failure before SharedMemory is created must not raise AttributeError
|
||||
or TypeError from cleanup.
|
||||
"""
|
||||
or TypeError from cleanup."""
|
||||
with (
|
||||
patch(f"{self._MOD}.HF3FS_AVAILABLE", True),
|
||||
patch(f"{self._MOD}.register_fd"),
|
||||
|
||||
@@ -212,8 +212,7 @@ class TestUpdateConnectorOutput:
|
||||
self, mock_connector
|
||||
):
|
||||
"""Test that method returns early when kv_cache_events is not
|
||||
LMCacheKVEvents.
|
||||
"""
|
||||
LMCacheKVEvents."""
|
||||
# Create a mock object that is not LMCacheKVEvents
|
||||
fake_events = MagicMock()
|
||||
connector_output = KVConnectorOutput(kv_cache_events=fake_events)
|
||||
|
||||
@@ -199,8 +199,7 @@ def _bare_worker() -> MooncakeConnectorWorker:
|
||||
"""Construct a MooncakeConnectorWorker skipping __init__ (full init requires
|
||||
a live TransferEngine). Only the attributes touched by the methods under
|
||||
test are populated; role flags and async_zmq_ctx keep __del__'s shutdown
|
||||
path a no-op.
|
||||
"""
|
||||
path a no-op."""
|
||||
worker = MooncakeConnectorWorker.__new__(MooncakeConnectorWorker)
|
||||
worker.xfer_stats = MooncakeKVConnectorStats()
|
||||
worker.engine = MagicMock()
|
||||
|
||||
@@ -77,8 +77,7 @@ def test_store_tp_does_not_override_backend_kv_cache_layout(extra_config):
|
||||
|
||||
def _make_kv_cache_config() -> KVCacheConfig:
|
||||
"""Single-group full-attention KVCacheConfig — enough for the connector
|
||||
constructor's validate() pass.
|
||||
"""
|
||||
constructor's validate() pass."""
|
||||
spec = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
|
||||
return KVCacheConfig(
|
||||
num_blocks=4,
|
||||
@@ -693,8 +692,7 @@ def _poll_lookup(client, req_id, num_tokens=128, block_hashes=(), timeout=5.0):
|
||||
|
||||
def _gated_recv(gate: threading.Event, value: int):
|
||||
"""Mock recv side-effect that blocks until ``gate`` is set, so the
|
||||
executor's lookup can be held pending deterministically.
|
||||
"""
|
||||
executor's lookup can be held pending deterministically."""
|
||||
|
||||
def recv():
|
||||
gate.wait()
|
||||
@@ -705,8 +703,7 @@ def _gated_recv(gate: threading.Event, value: int):
|
||||
|
||||
def test_lookup_key_client_non_block_lookup_async():
|
||||
"""Non-blocking lookup defers to the executor: None first, hit once the
|
||||
Future resolves.
|
||||
"""
|
||||
Future resolves."""
|
||||
vllm_config = _make_vllm_config()
|
||||
|
||||
with patch(
|
||||
|
||||
@@ -33,8 +33,7 @@ def _mamba_align(block_size=32):
|
||||
def _make_coord(groups, hash_block_size, use_eagle=False, retention_interval=None):
|
||||
"""Construct a coordinator using the natural LCM of group block sizes as
|
||||
the scheduler block size — mirrors ``resolve_kv_cache_block_sizes`` for
|
||||
the test fixtures.
|
||||
"""
|
||||
the test fixtures."""
|
||||
block_sizes = [g.kv_cache_spec.block_size for g in groups]
|
||||
scheduler_block_size = lcm(*block_sizes)
|
||||
return MooncakeStoreCoordinator(
|
||||
@@ -144,8 +143,7 @@ def test_coordinator_single_full_attention_no_hits():
|
||||
|
||||
def test_coordinator_single_swa_tautological_pool_masks_pre_window():
|
||||
"""SWA tautological-pool: hit_length spans full prefix, mask is
|
||||
tail-window only.
|
||||
"""
|
||||
tail-window only."""
|
||||
groups = [KVCacheGroupSpec(["L0"], _swa(block_size=16, sliding_window=32))]
|
||||
coord = _make_coord(groups, hash_block_size=16)
|
||||
hs = _hashes(4) # 4 chunks * 16 tokens
|
||||
@@ -192,8 +190,7 @@ def test_coordinator_hybrid_hole_in_full_clips_both():
|
||||
|
||||
def test_coordinator_group_block_size_double_hash():
|
||||
"""Group block_size=32 over hash_block_size=16 hashes: adjacent
|
||||
hashes merge before pool lookup.
|
||||
"""
|
||||
hashes merge before pool lookup."""
|
||||
groups = [
|
||||
KVCacheGroupSpec(["L0"], _full(16)),
|
||||
KVCacheGroupSpec(["L1"], _full(32)),
|
||||
@@ -217,8 +214,7 @@ def test_coordinator_group_block_size_double_hash():
|
||||
def test_coordinator_fine_grained_partial_tail_hit():
|
||||
"""K3 shape: FA + mamba-align, block_size=32 over hash_block_size=16. When
|
||||
both groups have the sub-block boundary hash, the reconciled hit lands on
|
||||
the hash boundary (48), not the block boundary (32).
|
||||
"""
|
||||
the hash boundary (48), not the block boundary (32)."""
|
||||
groups = [
|
||||
KVCacheGroupSpec(["L0"], _full(32)),
|
||||
KVCacheGroupSpec(["L1"], _mamba_align(32)),
|
||||
@@ -238,8 +234,7 @@ def test_coordinator_fine_grained_partial_tail_hit():
|
||||
|
||||
def test_coordinator_fine_grained_clips_when_one_group_missing_tail():
|
||||
"""If only one group has the sub-block boundary, min-convergence clips the
|
||||
reconciled hit back to the block boundary (32).
|
||||
"""
|
||||
reconciled hit back to the block boundary (32)."""
|
||||
groups = [
|
||||
KVCacheGroupSpec(["L0"], _full(32)),
|
||||
KVCacheGroupSpec(["L1"], _mamba_align(32)),
|
||||
@@ -280,8 +275,7 @@ def test_store_mask_swa_only_window_around_each_lcm_boundary():
|
||||
"""Hybrid full-attn(block=32) + SWA(block=8, sw=8). lcm=32. With
|
||||
aligned=64 the SWA group should mark exactly the blocks ending at 32
|
||||
and 64 (i.e. blocks 3 and 7 at block_size=8); the rest can never
|
||||
participate in any future hit.
|
||||
"""
|
||||
participate in any future hit."""
|
||||
full = _full(32)
|
||||
swa = _swa(block_size=8, sliding_window=8)
|
||||
groups = [KVCacheGroupSpec(["L0"], full), KVCacheGroupSpec(["L1"], swa)]
|
||||
@@ -327,8 +321,7 @@ def test_store_mask_swa_does_not_double_scale_under_dcp():
|
||||
|
||||
def test_store_mask_swa_wider_window_covers_more_blocks_per_lcm():
|
||||
"""Same hybrid layout but sliding_window=16 (= 2 SWA blocks). Each lcm
|
||||
boundary should now span two SWA tail blocks.
|
||||
"""
|
||||
boundary should now span two SWA tail blocks."""
|
||||
full = _full(32)
|
||||
swa = _swa(block_size=8, sliding_window=16)
|
||||
groups = [KVCacheGroupSpec(["L0"], full), KVCacheGroupSpec(["L1"], swa)]
|
||||
@@ -358,8 +351,7 @@ def test_store_mask_dsv4_5_groups_full_mla_plus_4_swa():
|
||||
"""DSV4-shaped: full-MLA(B=256) + 4 SWA groups with B in {64, 64, 4, 8}
|
||||
and varied sliding windows. lcm=256, hash_block_size=4. Two lcm segments
|
||||
(aligned_len=512). Validates that the tile-once strategy produces the
|
||||
expected per-segment tail-window pattern, repeated.
|
||||
"""
|
||||
expected per-segment tail-window pattern, repeated."""
|
||||
full_mla = _full(block_size=256)
|
||||
swa_64_sw128 = _swa(block_size=64, sliding_window=128)
|
||||
swa_64_sw512 = _swa(block_size=64, sliding_window=512)
|
||||
@@ -393,8 +385,7 @@ def test_store_mask_dsv4_5_groups_full_mla_plus_4_swa():
|
||||
|
||||
def test_store_mask_fast_path_all_block_sizes_equal_lcm():
|
||||
"""When every non-full-attn group already aligns to lcm_block_size, the
|
||||
fast path returns all-True without invoking find_longest_cache_hit.
|
||||
"""
|
||||
fast path returns all-True without invoking find_longest_cache_hit."""
|
||||
full = _full(block_size=64)
|
||||
swa = _swa(block_size=64, sliding_window=128)
|
||||
groups = [KVCacheGroupSpec(["L0"], full), KVCacheGroupSpec(["L1"], swa)]
|
||||
@@ -407,8 +398,7 @@ def test_store_mask_fast_path_all_block_sizes_equal_lcm():
|
||||
|
||||
def test_store_mask_fast_path_single_attention_group():
|
||||
"""Two groups sharing the same SWA spec collapse to one attention group;
|
||||
no lcm filter applies, every chunk is True.
|
||||
"""
|
||||
no lcm filter applies, every chunk is True."""
|
||||
swa = _swa(block_size=16, sliding_window=32)
|
||||
groups = [KVCacheGroupSpec(["L0"], swa), KVCacheGroupSpec(["L1"], swa)]
|
||||
coord = _make_coord(groups, hash_block_size=16)
|
||||
@@ -422,8 +412,7 @@ def test_store_mask_fast_path_single_attention_group():
|
||||
|
||||
def _retention_groups():
|
||||
"""Hybrid full-attn(block=32) + SWA(block=8, sw=8); lcm=32. The SWA group
|
||||
densely keeps one tail block per 32-token boundary.
|
||||
"""
|
||||
densely keeps one tail block per 32-token boundary."""
|
||||
full = _full(32)
|
||||
swa = _swa(block_size=8, sliding_window=8)
|
||||
return [KVCacheGroupSpec(["L0"], full), KVCacheGroupSpec(["L1"], swa)]
|
||||
@@ -431,8 +420,7 @@ def _retention_groups():
|
||||
|
||||
def test_store_mask_dense_default_matches_every_lcm_boundary():
|
||||
"""retention_interval=None (default) keeps the SWA tail at every lcm
|
||||
boundary: tokens 32/64/96/128 -> chunks 3/7/11/15.
|
||||
"""
|
||||
boundary: tokens 32/64/96/128 -> chunks 3/7/11/15."""
|
||||
coord = _make_coord(_retention_groups(), hash_block_size=8)
|
||||
masks = coord.store_mask(128)
|
||||
assert masks[0] is None
|
||||
@@ -442,8 +430,7 @@ def test_store_mask_dense_default_matches_every_lcm_boundary():
|
||||
def test_store_mask_retention_interval_sparsifies_swa_tails():
|
||||
"""retention_interval=64 keeps an SWA tail once per 64-token segment
|
||||
(chunks 7 and 15) instead of every 32 tokens, dropping the mid-segment
|
||||
boundaries at 32 and 96.
|
||||
"""
|
||||
boundaries at 32 and 96."""
|
||||
coord = _make_coord(_retention_groups(), hash_block_size=8, retention_interval=64)
|
||||
masks = coord.store_mask(128)
|
||||
assert masks[0] is None # full attn unaffected
|
||||
@@ -452,8 +439,7 @@ def test_store_mask_retention_interval_sparsifies_swa_tails():
|
||||
|
||||
def test_store_mask_retention_interval_zero_keeps_only_replay_boundary():
|
||||
"""retention_interval=0 drops all segment tails; only the latest replay
|
||||
boundary (capped at num_prompt-1, aligned down to lcm) is retained.
|
||||
"""
|
||||
boundary (capped at num_prompt-1, aligned down to lcm) is retained."""
|
||||
coord = _make_coord(_retention_groups(), hash_block_size=8, retention_interval=0)
|
||||
# No replay info -> nothing reachable for the SWA group.
|
||||
assert coord.store_mask(128)[1] == [False] * 16
|
||||
@@ -464,8 +450,7 @@ def test_store_mask_retention_interval_zero_keeps_only_replay_boundary():
|
||||
|
||||
def test_store_mask_retention_interval_keeps_segment_and_replay_tails():
|
||||
"""Sparse segment tails (interval=64 -> chunks 7,15) plus the replay
|
||||
boundary tail (num_prompt=100 -> chunk 11) coexist.
|
||||
"""
|
||||
boundary tail (num_prompt=100 -> chunk 11) coexist."""
|
||||
coord = _make_coord(_retention_groups(), hash_block_size=8, retention_interval=64)
|
||||
masks = coord.store_mask(128, num_prompt_tokens=100)
|
||||
assert masks[1] == [i in (7, 11, 15) for i in range(16)]
|
||||
@@ -503,8 +488,7 @@ def test_store_mask_excludes_mamba_groups_lookup_unaffected():
|
||||
nulled/freed; speculative blocks relocate), so the positional normal save
|
||||
must never cover mamba chunks — regardless of retention. Lookups still
|
||||
probe mamba boundaries: their keys come from the pinned snapshot/CoW
|
||||
hand-off path instead.
|
||||
"""
|
||||
hand-off path instead."""
|
||||
groups = [
|
||||
KVCacheGroupSpec(["L0"], _full(16)),
|
||||
KVCacheGroupSpec(["L1"], _mamba_align(16)),
|
||||
@@ -525,8 +509,7 @@ def test_store_mask_excludes_mamba_groups_lookup_unaffected():
|
||||
|
||||
def test_lookup_with_eagle_pops_last_full_attention_block():
|
||||
"""Sanity: with use_eagle, find_longest_cache_hit drops the last block.
|
||||
Pairs with the load_mask test below to lock the round-trip contract.
|
||||
"""
|
||||
Pairs with the load_mask test below to lock the round-trip contract."""
|
||||
groups = [KVCacheGroupSpec(["L0"], _full(16))]
|
||||
coord = _make_coord(groups, hash_block_size=16, use_eagle=True)
|
||||
hs = _hashes(4)
|
||||
@@ -567,8 +550,7 @@ def test_load_mask_with_eagle_does_not_double_prune_full_attention():
|
||||
def test_load_mask_with_eagle_hybrid_full_plus_swa():
|
||||
"""Hybrid (FullAttn + SWA) with eagle: load_mask must cover every chunk
|
||||
in [0, token_len) for the FullAttn group; SWA group keeps its
|
||||
tail-window mask.
|
||||
"""
|
||||
tail-window mask."""
|
||||
groups = [
|
||||
KVCacheGroupSpec(["L0"], _full(16)),
|
||||
KVCacheGroupSpec(["L1"], _swa(16, 32)),
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""End-to-end save->lookup test for MooncakeStoreConnector on a hybrid
|
||||
(SWA + Full) attention config, using a dict-backed mock store.
|
||||
"""
|
||||
(SWA + Full) attention config, using a dict-backed mock store."""
|
||||
|
||||
import sys
|
||||
import threading
|
||||
@@ -283,8 +282,7 @@ def test_e2e_swa_plus_full_save_then_lookup_hits():
|
||||
|
||||
def test_recv_skips_swa_blocks_before_window():
|
||||
"""Producer stored every block for both groups; consumer must only fetch
|
||||
SWA blocks within the sliding window, not the head.
|
||||
"""
|
||||
SWA blocks within the sliding window, not the head."""
|
||||
full = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
|
||||
# sliding_window=32, block_size=16 → 2 contiguous blocks within window.
|
||||
swa = SlidingWindowSpec(
|
||||
@@ -352,8 +350,7 @@ def test_recv_skips_swa_blocks_before_window():
|
||||
|
||||
def test_chunked_token_database_hash_block_size_smaller_than_block_size():
|
||||
"""DSv4-style: hash_block_size=4, group block_size=16 — process_tokens
|
||||
keys each chunk by its ending fine hash, including a partial tail.
|
||||
"""
|
||||
keys each chunk by its ending fine hash, including a partial tail."""
|
||||
md = KeyMetadata("m", 0, 0, 0, 0, group_id=3)
|
||||
db = ChunkedTokenDatabase(md, block_size=16, hash_block_size=4)
|
||||
db.set_kv_caches_base_addr([0])
|
||||
@@ -386,8 +383,7 @@ def test_sub_block_partial_tail_offload_reads_cow_block():
|
||||
"""Sub-block prompt (the 900/128/1536 shape, scaled to 12/4/16): the
|
||||
partial tail is offloaded for both groups under the boundary sub-hash. The
|
||||
full-attention block is read from the request block table; the mamba block
|
||||
is the core-provided CoW target, not block_ids.
|
||||
"""
|
||||
is the core-provided CoW target, not block_ids."""
|
||||
full = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
|
||||
mamba = MambaSpec(
|
||||
block_size=16,
|
||||
@@ -466,8 +462,7 @@ def test_sub_block_partial_tail_offload_reads_cow_block():
|
||||
def test_offload_syncs_event_before_put():
|
||||
"""An offload-carrying meta synchronizes its CoW-fence event before the
|
||||
store put reads the blocks, then completes in one pass and drains the
|
||||
completion counter.
|
||||
"""
|
||||
completion counter."""
|
||||
full = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
|
||||
mamba = MambaSpec(
|
||||
block_size=16,
|
||||
@@ -538,8 +533,7 @@ def test_sub_block_partial_tail_offload_covers_smaller_group_blocks():
|
||||
offload must persist every FA block up to the boundary — the normal save
|
||||
floors to the lcm, so those blocks are otherwise never written and the
|
||||
consumer's per-group lookup would miss. The mamba boundary block still
|
||||
reads the core-provided CoW target.
|
||||
"""
|
||||
reads the core-provided CoW target."""
|
||||
full = FullAttentionSpec(block_size=4, num_kv_heads=8, head_size=64, dtype=None)
|
||||
mamba = MambaSpec(
|
||||
block_size=16,
|
||||
@@ -810,8 +804,7 @@ def test_ring_scratch_group_is_never_stored_and_does_not_block_hits():
|
||||
"""A per-request ring group (CircularBufferSpec, capacity as block_size)
|
||||
sits beside the paged group: worker setup tolerates its block size (the
|
||||
DeepSeek-V4.1 compressor ring is 8 rows), saving a request stores paged
|
||||
blocks only, and lookup hits on the paged group alone.
|
||||
"""
|
||||
blocks only, and lookup hits on the paged group alone."""
|
||||
full = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
|
||||
ring = CircularBufferSpec(
|
||||
block_size=8, num_kv_heads=1, head_size=64, head_size_v=0, dtype=torch.uint8
|
||||
|
||||
@@ -536,8 +536,7 @@ def _make_pending_load_unfinished_request(
|
||||
|
||||
def _make_pending_load_scheduler_output() -> SimpleNamespace:
|
||||
"""scheduler_output for a step where req-0 is parked on a pending load
|
||||
(not in scheduled_new_reqs or scheduled_cached_reqs).
|
||||
"""
|
||||
(not in scheduled_new_reqs or scheduled_cached_reqs)."""
|
||||
return SimpleNamespace(
|
||||
finished_req_ids=set(),
|
||||
preempted_req_ids=set(),
|
||||
@@ -713,8 +712,7 @@ def test_running_request_not_in_resumed_req_ids_appends_blocks():
|
||||
def test_resumed_request_in_resumed_req_ids_replaces_blocks():
|
||||
"""A request the scheduler marks resumed gets the FULL block table in
|
||||
new_block_ids and must REPLACE the tracker's blocks (not append), even if
|
||||
a stale tracker from before preemption is still present.
|
||||
"""
|
||||
a stale tracker from before preemption is still present."""
|
||||
scheduler = _make_bare_scheduler()
|
||||
_make_resumed_unfinished_request(
|
||||
scheduler,
|
||||
|
||||
@@ -432,8 +432,7 @@ def _patch_worker_runtime(
|
||||
|
||||
def test_pool_key_to_string_without_prefix_is_unchanged():
|
||||
"""Default (empty) cache_prefix keeps keys byte-identical to the
|
||||
historical unprefixed format so existing deployments keep their hits.
|
||||
"""
|
||||
historical unprefixed format so existing deployments keep their hits."""
|
||||
key = PoolKey(KeyMetadata("test-model", 0, 0, 0, 0), "deadbeef")
|
||||
assert (
|
||||
key.to_string() == "test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@deadbeef"
|
||||
@@ -574,8 +573,7 @@ def test_tp_shared_receiving_reads_each_local_store_shard():
|
||||
|
||||
def test_pool_key_cache_prefix_namespaces_and_disambiguates():
|
||||
"""A non-empty cache_prefix is prepended, and two instances with
|
||||
different prefixes never collide on identical block hashes.
|
||||
"""
|
||||
different prefixes never collide on identical block hashes."""
|
||||
md_a = KeyMetadata("test-model", 0, 0, 0, 0, cache_prefix="depA")
|
||||
md_b = KeyMetadata("test-model", 0, 0, 0, 0, cache_prefix="depB")
|
||||
|
||||
@@ -591,8 +589,7 @@ def test_pool_key_cache_prefix_namespaces_and_disambiguates():
|
||||
|
||||
def test_default_local_buffer_size_matches_pr40900():
|
||||
"""PR-40900 shipped a 4 GiB default for local_buffer_size; the dual-mode
|
||||
patch preserves it (and the JSON key) so unchanged PR-40900 configs work.
|
||||
"""
|
||||
patch preserves it (and the JSON key) so unchanged PR-40900 configs work."""
|
||||
assert worker.DEFAULT_LOCAL_BUFFER_SIZE == 4 * 1024**3
|
||||
|
||||
|
||||
@@ -1014,8 +1011,7 @@ def test_store_sending_thread_skips_null_sparse_group_blocks():
|
||||
|
||||
def test_partial_tail_offload_skips_cap_omitted_mamba_group():
|
||||
"""A Mamba group omitted by the handoff cap must not fall back to the
|
||||
connector's positional block table.
|
||||
"""
|
||||
connector's positional block table."""
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheGroupSpec,
|
||||
@@ -1143,8 +1139,7 @@ def test_partial_tail_put_failure_activates_pressure_gate():
|
||||
def test_normal_save_excludes_mamba_group_and_null_blocks():
|
||||
"""The positional normal save must never cover mamba chunks (align-mode
|
||||
block tables are not append-only) and must skip chunks whose block is the
|
||||
reserved null block, for any group.
|
||||
"""
|
||||
reserved null block, for any group."""
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheGroupSpec,
|
||||
@@ -1205,8 +1200,7 @@ def test_normal_save_excludes_mamba_group_and_null_blocks():
|
||||
def test_block_aligned_snapshot_offload_uses_provided_block():
|
||||
"""A block-aligned hand-off (sparse-retention mamba checkpoint) uploads
|
||||
exactly the core-pinned block under the boundary-end hash key; the
|
||||
positional block table entry for that chunk must be ignored.
|
||||
"""
|
||||
positional block table entry for that chunk must be ignored."""
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
|
||||
store.batch_put_from_multi_buffers.side_effect = lambda keys, *a: [256] * len(keys)
|
||||
@@ -1234,8 +1228,7 @@ def test_block_aligned_snapshot_offload_uses_provided_block():
|
||||
def test_mixed_snapshot_and_sub_block_offloads():
|
||||
"""A retention snapshot and the prompt-end sub-block CoW tail can arrive
|
||||
in one hand-off; the sub-block path covers FA gap blocks but reads the
|
||||
mamba boundary only from the CoW block, never positionally.
|
||||
"""
|
||||
mamba boundary only from the CoW block, never positionally."""
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
|
||||
store.batch_put_from_multi_buffers.side_effect = lambda keys, *a: [256] * len(keys)
|
||||
@@ -1275,8 +1268,7 @@ def test_mixed_snapshot_and_sub_block_offloads():
|
||||
|
||||
def test_snapshot_offload_skips_null_handoff_block():
|
||||
"""A hand-off the core could not materialize (null block) carries no
|
||||
committed state; persisting it would poison the boundary key.
|
||||
"""
|
||||
committed state; persisting it would poison the boundary key."""
|
||||
store = MagicMock()
|
||||
thread = _make_partial_tail_send_thread(store)
|
||||
|
||||
@@ -1604,8 +1596,7 @@ def test_store_sending_thread_passes_replicate_config_when_preferred_segment_set
|
||||
def test_store_sending_thread_passes_default_replicate_config_when_no_preferred_segment(): # noqa: E501
|
||||
"""Without a preferred_segment the SendingThread still forwards a
|
||||
(default-constructed) ReplicateConfig so the C++ side always sees a
|
||||
well-defined config object.
|
||||
"""
|
||||
well-defined config object."""
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
|
||||
store.batch_put_from_multi_buffers.return_value = [256, 256]
|
||||
@@ -2115,8 +2106,7 @@ def test_recv_thread_stops_after_first_failing_disk_offload_sub_batch():
|
||||
|
||||
def test_recv_thread_skips_split_when_budget_holds_all_keys():
|
||||
"""PR-36 removed the count-based split trigger; with budget for 3 keys,
|
||||
all three should be requested in a single call.
|
||||
"""
|
||||
all three should be requested in a single call."""
|
||||
store = MagicMock()
|
||||
store.batch_get_into_multi_buffers.return_value = [256, 256, 256]
|
||||
thread = _make_store_recving_thread(
|
||||
@@ -2330,8 +2320,7 @@ def test_requester_worker_init_skips_disk_budget_when_offload_disabled(
|
||||
monkeypatch,
|
||||
):
|
||||
"""enable_offload=False zeroes out the disk budget so we don't generate
|
||||
redundant owner GET-RPCs.
|
||||
"""
|
||||
redundant owner GET-RPCs."""
|
||||
store = MagicMock()
|
||||
store.setup.return_value = 0
|
||||
_install_fake_mooncake(monkeypatch, store)
|
||||
@@ -3006,8 +2995,7 @@ def test_requester_worker_group_semantics_string_true_enables(
|
||||
|
||||
def test_store_sending_thread_clamps_token_len_to_lcm():
|
||||
"""Partial chunks past the last lcm boundary aren't stored — cache hits
|
||||
are always lcm-aligned (mirrors HybridKVCacheCoordinator).
|
||||
"""
|
||||
are always lcm-aligned (mirrors HybridKVCacheCoordinator)."""
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.return_value = [0, 0]
|
||||
store.batch_put_from_multi_buffers.return_value = [256, 256]
|
||||
@@ -3032,8 +3020,7 @@ def test_store_sending_thread_clamps_token_len_to_lcm():
|
||||
|
||||
def test_store_sending_thread_skips_when_token_len_below_lcm():
|
||||
"""Requests shorter than lcm_block_size cannot produce any aligned chunk,
|
||||
so neither the existence check nor the put should be issued.
|
||||
"""
|
||||
so neither the existence check nor the put should be issued."""
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheGroupSpec
|
||||
|
||||
store = MagicMock()
|
||||
@@ -3823,8 +3810,7 @@ def test_lookup_full_hit_with_eagle_pops_once_not_twice():
|
||||
"""Eagle already leaves the last block for the drafter, so a
|
||||
full-prompt re-derivation must never fire for eagle-governed hits:
|
||||
firing would anchor the search one block lower and pop a second
|
||||
block, regressing the hit by an extra producer boundary.
|
||||
"""
|
||||
block, regressing the hit by an extra producer boundary."""
|
||||
worker = _make_bare_worker(block_size=16)
|
||||
worker.coord = mooncake_store_worker.MooncakeStoreCoordinator(
|
||||
worker._kv_cache_groups,
|
||||
@@ -3981,8 +3967,7 @@ def test_lookup_full_hit_swa_degrades_when_no_stored_boundary_is_usable():
|
||||
the full hit into 48, whose SWA window needs the never-written block 1,
|
||||
so every load failed and the recompute re-entered the same lookup. The
|
||||
re-derivation must report that no stored boundary below the request end
|
||||
is usable.
|
||||
"""
|
||||
is usable."""
|
||||
from vllm.v1.kv_cache_interface import KVCacheGroupSpec, SlidingWindowSpec
|
||||
|
||||
worker = _make_bare_worker(block_size=16)
|
||||
@@ -4004,8 +3989,7 @@ def test_lookup_full_hit_swa_degrades_when_no_stored_boundary_is_usable():
|
||||
def test_lookup_swa_single_group_returns_full_when_tail_window_present():
|
||||
"""Single-SWA, sliding_window=32 (= 2 blocks): producer stored only the
|
||||
tail. Coordinator-driven lookup returns full prefix even though the
|
||||
pre-window blocks are absent.
|
||||
"""
|
||||
pre-window blocks are absent."""
|
||||
from vllm.v1.kv_cache_interface import KVCacheGroupSpec, SlidingWindowSpec
|
||||
|
||||
worker = _make_bare_worker(block_size=16)
|
||||
@@ -4401,8 +4385,7 @@ def test_config_defaults_to_embedded():
|
||||
|
||||
def test_config_pr40900_unchanged(tmp_path):
|
||||
"""A literal PR-40900 config (no mode, no enable_offload, no preferred_segment)
|
||||
parses without raising and resolves to embedded mode.
|
||||
"""
|
||||
parses without raising and resolves to embedded mode."""
|
||||
config_path = _write_mooncake_config(
|
||||
tmp_path,
|
||||
{
|
||||
@@ -4510,8 +4493,7 @@ def test_config_zero_local_buffer():
|
||||
def test_topology_standalone_store_with_disk_offload(tmp_path, monkeypatch):
|
||||
"""standalone-store + disk: global_segment_size=0, enable_offload=True,
|
||||
preferred_segment set. Assert setup() positional args, ReplicateConfig
|
||||
wiring, and that the disk-offload buffer budget is allocated.
|
||||
"""
|
||||
wiring, and that the disk-offload buffer budget is allocated."""
|
||||
store = MagicMock()
|
||||
store.setup.return_value = 0
|
||||
fake_replicate_config_cls = _install_fake_mooncake(monkeypatch, store)
|
||||
@@ -4560,8 +4542,7 @@ def test_topology_standalone_store_with_disk_offload(tmp_path, monkeypatch):
|
||||
def test_topology_embedded_cpu_only(tmp_path, monkeypatch):
|
||||
"""Embedded + CPU-only: no mode key (defaults to embedded),
|
||||
global_segment_size>0, enable_offload absent, no preferred_segment.
|
||||
This is the PR-40900 baseline recipe.
|
||||
"""
|
||||
This is the PR-40900 baseline recipe."""
|
||||
store = MagicMock()
|
||||
store.setup.return_value = 0
|
||||
fake_replicate_config_cls = _install_fake_mooncake(monkeypatch, store)
|
||||
@@ -4750,8 +4731,7 @@ def test_store_worker_close_swallows_store_errors():
|
||||
def test_blob_block_hashes_wire_roundtrip():
|
||||
"""The lookup wire format sends a ``hash_len`` frame plus the raw hashes
|
||||
concatenated back-to-back; the server rebuilds them through a zero-copy
|
||||
``BlobBlockHashes`` view over the frame buffer.
|
||||
"""
|
||||
``BlobBlockHashes`` view over the frame buffer."""
|
||||
hashes = [BlockHash(bytes([i]) * 16) for i in range(5)]
|
||||
hash_len = len(hashes[0])
|
||||
|
||||
|
||||
@@ -692,8 +692,7 @@ def test_moriio_handshake_returns_metadata(mock_parallel_groups):
|
||||
def test_resolve_host_ip_prefers_extra_config():
|
||||
"""An explicit ``host_ip`` in kv_connector_extra_config overrides get_ip()
|
||||
(so an external router can advertise a routable/internal address); an
|
||||
absent or empty value falls back to get_ip().
|
||||
"""
|
||||
absent or empty value falls back to get_ip()."""
|
||||
assert resolve_host_ip({"host_ip": "10.0.0.7"}) == "10.0.0.7"
|
||||
|
||||
fallback = get_ip()
|
||||
@@ -747,8 +746,7 @@ def _read_scheduler(
|
||||
|
||||
def test_hma_blocks_per_sw_two_groups():
|
||||
"""A Full + SlidingWindow config runs with HMA and computes block budgets
|
||||
correctly
|
||||
"""
|
||||
correctly"""
|
||||
scheduler = _read_scheduler(_make_hybrid_kv_cache_config())
|
||||
assert scheduler._is_hma_required is True
|
||||
# cdiv(32, 16) + 1 == 3 for the sliding-window group, 0 for full attention.
|
||||
@@ -766,8 +764,7 @@ def test_hma_blocks_per_sw_two_groups():
|
||||
def test_is_hma_required(swa_enabled, disable_hma, expected_is_hma):
|
||||
"""_is_hma_required tracks both the KV cache groups and the
|
||||
--disable-hybrid-kv-cache-manager flag. When HMA is off,
|
||||
get_exchange_clipped_blocks must be a no-op.
|
||||
"""
|
||||
get_exchange_clipped_blocks must be a no-op."""
|
||||
config = (
|
||||
_make_hybrid_kv_cache_config() if swa_enabled else _make_test_kv_cache_config()
|
||||
)
|
||||
@@ -780,8 +777,7 @@ def test_is_hma_required(swa_enabled, disable_hma, expected_is_hma):
|
||||
|
||||
def test_non_sliding_window_hybrid_is_rejected():
|
||||
"""A hybrid group that is not sliding-window (e.g. chunked-local
|
||||
attention) must fail closed rather than be silently mistransferred.
|
||||
"""
|
||||
attention) must fail closed rather than be silently mistransferred."""
|
||||
full_spec = FullAttentionSpec(
|
||||
block_size=16, num_kv_heads=4, head_size=64, dtype=torch.float16
|
||||
)
|
||||
@@ -815,8 +811,7 @@ def test_non_sliding_window_hybrid_is_rejected():
|
||||
|
||||
def test_token_count_basis_uses_full_attention_group():
|
||||
"""Chunked-prefill token counting must use an unclipped full-attention
|
||||
group (blocks_per_sw == 0), not a clipped sliding-window group.
|
||||
"""
|
||||
group (blocks_per_sw == 0), not a clipped sliding-window group."""
|
||||
scheduler = _read_scheduler(_make_hybrid_kv_cache_config())
|
||||
# Group 0 is the full-attention group
|
||||
assert scheduler._full_attn_group_idx == 0
|
||||
@@ -825,8 +820,7 @@ def test_token_count_basis_uses_full_attention_group():
|
||||
|
||||
def test_get_exchange_clipped_blocks_clips_only_sw_group():
|
||||
"""get_exchange_clipped_blocks keeps the full attn group intact and clips the
|
||||
sliding-window group to its window tail.
|
||||
"""
|
||||
sliding-window group to its window tail."""
|
||||
scheduler = _read_scheduler(_make_hybrid_kv_cache_config())
|
||||
full = [10, 11, 12, 13, 14]
|
||||
sw = [20, 21, 22, 23, 24]
|
||||
@@ -837,8 +831,7 @@ def test_get_exchange_clipped_blocks_clips_only_sw_group():
|
||||
|
||||
def test_metadata_hma_block_ids_preserved_per_group():
|
||||
"""add_new_req stores per-group (BlockIds) block lists unchanged for both
|
||||
read and write, so the hybrid group structure is retained.
|
||||
"""
|
||||
read and write, so the hybrid group structure is retained."""
|
||||
metadata = MoRIIOConnectorMetadata()
|
||||
|
||||
# Assume:
|
||||
|
||||
@@ -110,8 +110,7 @@ def make_meta(*, p_tp: int, p_dp: int, remote_dp_rank: int, host: str = "phost0"
|
||||
def build_decode_workers(cfg: PDConfig) -> list:
|
||||
"""Decode workers that issue reads for one prefill instance. A TP decode
|
||||
instance reads from every tp rank; a DP decode instance from each dp-rank
|
||||
worker. Reused across requests so per-worker round-robin state advances.
|
||||
"""
|
||||
worker. Reused across requests so per-worker round-robin state advances."""
|
||||
if cfg.d_tp > 1:
|
||||
return [
|
||||
make_decode_worker(world_size=cfg.d_tp, tp_rank=r, dp_rank=0)
|
||||
@@ -126,8 +125,7 @@ def prefill_target_multiset(cfg: PDConfig, rounds: int) -> Counter:
|
||||
"""Drive the REAL _resolve_read_source over a fair input stream; tally which
|
||||
prefill GPU each read targets. The only modelled assumption is the proxy
|
||||
contract -- each owner dp-rank delivered equally (``for owner_dp in
|
||||
range(p_dp)``); no proxy algorithm is reproduced.
|
||||
"""
|
||||
range(p_dp)``); no proxy algorithm is reproduced."""
|
||||
workers = build_decode_workers(cfg)
|
||||
hits: Counter = Counter()
|
||||
for _ in range(rounds):
|
||||
|
||||
@@ -555,8 +555,7 @@ class TestMultiConnectorStats:
|
||||
|
||||
def test_build_kv_connector_stats_reconstructs_nixl_stats(self):
|
||||
"""Test that NixlConnector stats are properly reconstructed with
|
||||
correct data.
|
||||
"""
|
||||
correct data."""
|
||||
serialized_data = {
|
||||
"NixlConnector": {
|
||||
"transfer_duration": [1.5, 2.3],
|
||||
|
||||
@@ -318,8 +318,7 @@ def test_abort_immediately_remote_prefill_enqueues_empty_recv():
|
||||
"""A remote-prefill request added with abort_immediately=True should
|
||||
be added to the scheduler's waiting queue then immediately aborted, so the
|
||||
NIXL connector's request_finished hook enqueues an empty recv to notify
|
||||
the prefill instance to free its blocks.
|
||||
"""
|
||||
the prefill instance to free its blocks."""
|
||||
from vllm.v1.request import RequestStatus
|
||||
|
||||
scheduler = create_scheduler(create_vllm_config())
|
||||
@@ -3146,8 +3145,7 @@ def test_split_read_failure_defers_report_until_last_handle(
|
||||
request — nor invalidate its blocks — while the sibling xfer is still in
|
||||
flight: a posted READ cannot be aborted and would DMA into blocks the
|
||||
scheduler could free and reuse. The report happens exactly once, when the
|
||||
last handle is terminal.
|
||||
"""
|
||||
last handle is terminal."""
|
||||
request_id = "split_read_partial_failure"
|
||||
err_handle, live_handle = 11, 22
|
||||
connector, worker, wrapper = _make_split_read_connector(
|
||||
|
||||
@@ -281,8 +281,7 @@ def test_update_state_after_alloc_tracks_cached_blocks_per_group():
|
||||
"""Hybrid SWA+FA requests can have different prefix-cache-hit counts per
|
||||
KV cache group. Each group's count must be tracked independently rather
|
||||
than collapsed to a single scalar, or DCP read-phase alignment
|
||||
(local_num_computed_blocks) would misalign one of the groups.
|
||||
"""
|
||||
(local_num_computed_blocks) would misalign one of the groups."""
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.pull_scheduler import (
|
||||
NixlPullConnectorScheduler,
|
||||
)
|
||||
@@ -336,8 +335,7 @@ def test_full_local_hit_is_not_awaited_by_the_scheduler():
|
||||
owed to the producer -- so the worker must not report it in
|
||||
finished_recving. Doing so trips `assert RequestStatus.is_finished` in
|
||||
_update_from_kv_xfer_finished, which only tolerates a finished recv for a
|
||||
request that is parked or already done.
|
||||
"""
|
||||
request that is parked or already done."""
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.pull_scheduler import (
|
||||
NixlPullConnectorScheduler,
|
||||
)
|
||||
@@ -442,8 +440,7 @@ def test_apply_dcp_prefix_caching_matches_global_logical_positions(
|
||||
"""_apply_dcp_prefix_caching must operate on logical block IDs (DCP
|
||||
interleaves at logical-block granularity) and always return equal-length
|
||||
local/remote slices, so a later, unconditional _apply_prefix_caching
|
||||
trim is a guaranteed no-op rather than double-processing the read.
|
||||
"""
|
||||
trim is a guaranteed no-op rather than double-processing the read."""
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import (
|
||||
NixlConnectorWorker,
|
||||
)
|
||||
@@ -842,8 +839,7 @@ def test_apply_prefix_caching_ssm_prefix_cache_hit(
|
||||
def test_apply_prefix_caching_ssm_unpairable_slots_rejected():
|
||||
"""Local SSM slots can only exceed the remote ones by the position D
|
||||
recomputes itself. A larger excess means the lists aren't
|
||||
position-aligned: fail loudly rather than transfer into wrong slots.
|
||||
"""
|
||||
position-aligned: fail loudly rather than transfer into wrong slots."""
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import (
|
||||
NixlConnectorWorker,
|
||||
)
|
||||
@@ -1159,8 +1155,7 @@ def test_nixl_region_sort_preserves_pipeline_layer_order():
|
||||
@pytest.mark.cpu_test
|
||||
def test_get_block_descs_ids_hybrid_ssm():
|
||||
"""Test _compute_desc_ids uses per-group strides for hybrid
|
||||
FA+SSM when ratio=1 (no kernel block size mismatch).
|
||||
"""
|
||||
FA+SSM when ratio=1 (no kernel block size mismatch)."""
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec, MambaSpec
|
||||
|
||||
worker = _make_mock_worker_for_desc_ids(
|
||||
@@ -1244,8 +1239,7 @@ def test_get_block_descs_ids_uses_per_region_pool_capacity():
|
||||
@pytest.mark.cpu_test
|
||||
def test_get_block_descs_ids_kernel_block_mismatch():
|
||||
"""Test _compute_desc_ids uses different strides for FA
|
||||
(kernel blocks) vs SSM (logical blocks) when ratio > 1.
|
||||
"""
|
||||
(kernel blocks) vs SSM (logical blocks) when ratio > 1."""
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec, MambaSpec
|
||||
|
||||
ratio = 4
|
||||
@@ -1276,8 +1270,7 @@ def test_get_block_descs_ids_kernel_block_mismatch():
|
||||
def test_get_block_descs_ids_hetero_block_size_hybrid():
|
||||
"""With a block-size ratio, FA desc ids are ratio-expanded while SSM
|
||||
desc ids keep the unexpanded logical stride (state blocks are never
|
||||
sub-split).
|
||||
"""
|
||||
sub-split)."""
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec, MambaSpec
|
||||
|
||||
worker = _make_mock_worker_for_desc_ids(
|
||||
@@ -1316,8 +1309,7 @@ def _bind_worker_method(worker, name):
|
||||
@pytest.mark.cpu_test
|
||||
def test_map_block_ids_for_block_size_ratio_hybrid():
|
||||
"""Attention groups expand to remote granularity and clip to the remote
|
||||
coverage; mamba state blocks pass through 1:1.
|
||||
"""
|
||||
coverage; mamba state blocks pass through 1:1."""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import (
|
||||
@@ -1348,8 +1340,7 @@ def test_map_block_ids_for_block_size_ratio_hybrid():
|
||||
@pytest.mark.cpu_test
|
||||
def test_post_process_zeroes_untransferred_tail():
|
||||
"""The untransferred sub-blocks of the last local block are zeroed on
|
||||
receive; mamba state caches are untouched by the attention permute.
|
||||
"""
|
||||
receive; mamba state caches are untouched by the attention permute."""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import (
|
||||
@@ -1392,8 +1383,7 @@ def test_post_process_zeroes_untransferred_tail():
|
||||
@pytest.mark.cpu_test
|
||||
def test_nixl_metadata_hybrid_ssm_block_ids():
|
||||
"""Test NixlConnectorMetadata correctly stores block IDs for FA + SSM
|
||||
groups with different block counts (kernel mismatch active).
|
||||
"""
|
||||
groups with different block counts (kernel mismatch active)."""
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
|
||||
NixlConnectorMetadata,
|
||||
)
|
||||
@@ -1474,8 +1464,7 @@ def _make_fake_kv_cache_manager():
|
||||
@pytest.mark.cpu_test
|
||||
def test_zeroing_block_ids_cover_only_loaded_attention_blocks():
|
||||
"""Only zero-recorded (attention) groups contribute, sliced to the
|
||||
externally-loaded token range; Mamba state blocks are never zeroed.
|
||||
"""
|
||||
externally-loaded token range; Mamba state blocks are never zeroed."""
|
||||
manager = _make_fake_kv_cache_manager()
|
||||
|
||||
# Tokens [0, 16) are locally cached; the load covers tokens [16, 56).
|
||||
@@ -1503,8 +1492,7 @@ def test_scheduler_filters_connector_loaded_blocks_from_zeroing():
|
||||
@pytest.mark.cpu_test
|
||||
def test_failed_load_rezeroes_unwritten_skipped_blocks():
|
||||
"""A failed async load leaves zeroing-skipped blocks unwritten beyond
|
||||
the valid prefix; they must be zeroed before local recompute.
|
||||
"""
|
||||
the valid prefix; they must be zeroed before local recompute."""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from vllm.v1.core.sched.scheduler import Scheduler
|
||||
@@ -1964,8 +1952,7 @@ def test_logical_to_kernel_block_ids_with_remote_ratio(
|
||||
def test_exchange_clipped_blocks_ssm_single_state():
|
||||
"""In single-state cache modes, SSM lists are reduced to the running
|
||||
state slot: speculative scratch slots, null placeholders and the previous
|
||||
step's state carry nothing. Attention groups pass through untouched.
|
||||
"""
|
||||
step's state carry nothing. Attention groups pass through untouched."""
|
||||
sched = make_nixl_scheduler(has_mamba=True, is_hma_required=True)
|
||||
sched.blocks_per_sw = [0, 0]
|
||||
sched._ssm_spec_blocks = [None, 2]
|
||||
@@ -1992,8 +1979,7 @@ def test_exchange_clipped_blocks_ssm_single_state():
|
||||
@pytest.mark.cpu_test
|
||||
def test_exchange_clipped_blocks_ssm_positional_states():
|
||||
"""In "all" mode every position holds a state, so only the speculative
|
||||
slots go; placeholders stay to keep the list position-indexed.
|
||||
"""
|
||||
slots go; placeholders stay to keep the list position-indexed."""
|
||||
sched = make_nixl_scheduler(has_mamba=True, is_hma_required=True)
|
||||
sched.blocks_per_sw = [0, 0]
|
||||
sched._ssm_spec_blocks = [None, 2]
|
||||
@@ -2010,8 +1996,7 @@ def _make_hybrid_mla_kv_cache_config(num_blocks: int = 4):
|
||||
"""KimiLinear-shaped config: one MLA group and two KDA (GDN-typed
|
||||
MambaSpec) groups whose layers share the same HMA tensors, with a
|
||||
mamba-aligned unified page and an MLA kernel block smaller than the
|
||||
logical block.
|
||||
"""
|
||||
logical block."""
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
KVCacheConfig,
|
||||
@@ -2151,8 +2136,7 @@ def test_register_kv_caches_hybrid_mla_dual_purpose_regions():
|
||||
"""Hybrid MLA+KDA registration: HMA tensors shared by both layer types
|
||||
must be flagged as MLA regions even when a KDA layer registers them
|
||||
first, expose TP-independent kernel-granularity block lens, and build
|
||||
FA + mamba descriptors for every region.
|
||||
"""
|
||||
FA + mamba descriptors for every region."""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from vllm.config import set_current_vllm_config
|
||||
@@ -2253,8 +2237,7 @@ def test_register_kv_caches_hybrid_mla_dual_purpose_regions():
|
||||
def test_push_write_hybrid_mla_replicates_attention():
|
||||
"""Hybrid MLA+SSM push with P_TP < D_TP: attention blocks must be
|
||||
written to every covered D rank (replicated MLA latent) while SSM state
|
||||
is written per-rank through the split handles.
|
||||
"""
|
||||
is written per-rank through the split handles."""
|
||||
import threading
|
||||
from collections import defaultdict
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
@@ -24,8 +24,7 @@ from .utils import create_vllm_config
|
||||
|
||||
class _RecordingNixl:
|
||||
"""Minimal NIXL wrapper stand-in that records descriptor lists and
|
||||
prepared transfers so tests can resolve desc ids to byte ranges.
|
||||
"""
|
||||
prepared transfers so tests can resolve desc ids to byte ranges."""
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.dlists: dict[int, np.ndarray] = {}
|
||||
@@ -594,8 +593,7 @@ def _register_remote_agents(worker, metadata, tp_size):
|
||||
|
||||
def _owned_byte_ranges(worker, group_logical_ids):
|
||||
"""Byte ranges owned by a request: for each HMA region tensor, every
|
||||
logical block id of every group maps to one unified page.
|
||||
"""
|
||||
logical block id of every group maps to one unified page."""
|
||||
unified_page = worker._test_unified_page
|
||||
bases = [t.data_ptr() for t in worker._test_tensors]
|
||||
owned = []
|
||||
@@ -633,8 +631,7 @@ def test_hetero_ppl_multi_read_writes_stay_within_request_blocks():
|
||||
(kernel 4, ppl=3) vs remote (P, TP2) logical blocks of 8 tokens (ppl=2),
|
||||
equal kernel pages, tp_ratio=-2 multi-read with replicated MLA and
|
||||
TP-sharded KDA state. Every local descriptor of the request's reads must
|
||||
stay within its own blocks.
|
||||
"""
|
||||
stay within its own blocks."""
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
|
||||
NixlConnectorMetadata,
|
||||
)
|
||||
@@ -701,8 +698,7 @@ def _resolve(
|
||||
"""Resolve a desc id to (region, kind, token_start) where kind is 'attn'
|
||||
(desc-page sized, sub-block-aligned, in the request's attention blocks)
|
||||
or 'mamba'. token_start is the request-relative token offset, so local
|
||||
and remote are comparable even when their kernel blocks differ in size.
|
||||
"""
|
||||
and remote are comparable even when their kernel blocks differ in size."""
|
||||
addr, length, _ = (int(x) for x in desc_arr[int(idx)])
|
||||
for region, base in enumerate(bases):
|
||||
off = addr - base
|
||||
@@ -896,8 +892,7 @@ def _run_hetero_case(
|
||||
def test_hetero_ppl_token_alignment_sweep(local_block, remote_block, num_tokens):
|
||||
"""Sweep prompt lengths across block-boundary residues for several
|
||||
hetero-ppl geometries; assert neighbor-safety, token alignment, and
|
||||
coverage of every transferred kernel block.
|
||||
"""
|
||||
coverage of every transferred kernel block."""
|
||||
_run_hetero_case(
|
||||
local_block, kernel=4, remote_block=remote_block, num_tokens=num_tokens
|
||||
)
|
||||
@@ -916,8 +911,7 @@ def test_hetero_ppl_with_block_size_ratio(num_tokens):
|
||||
2). The transfer is clipped at remote sub-block granularity by the
|
||||
pairing and front-trimmed by _apply_prefix_caching, so the
|
||||
untransferred tail can span both a partial block and whole blocks —
|
||||
the case each of the two former zeroing paths handled only half of.
|
||||
"""
|
||||
the case each of the two former zeroing paths handled only half of."""
|
||||
_run_hetero_case(
|
||||
local_block=24,
|
||||
kernel=8,
|
||||
@@ -966,8 +960,7 @@ def test_mla_hybrid_large_ppl_geometry(num_tokens):
|
||||
"""KimiLinear-scale MLA-hybrid geometry (TP8 prefill -> TP1 decode):
|
||||
decode (local) logical block 5760 / kernel 64 (ppl=90), prefill
|
||||
(remote) logical block 768 (ppl=12), tp_ratio=-8 multi-read with
|
||||
replicated MLA and 8-way TP-sharded KDA state.
|
||||
"""
|
||||
replicated MLA and 8-way TP-sharded KDA state."""
|
||||
_run_hetero_case(
|
||||
local_block=5760,
|
||||
kernel=64,
|
||||
@@ -981,8 +974,7 @@ def test_mla_hybrid_large_ppl_geometry(num_tokens):
|
||||
def test_mismatched_mla_kernel_page_rejected_for_mla_hybrid():
|
||||
"""The MLA per-token page is TP-independent, so kernel block lengths
|
||||
differing by anything other than the block-size ratio must fail the
|
||||
handshake loudly rather than transfer at mismatched geometry.
|
||||
"""
|
||||
handshake loudly rather than transfer at mismatched geometry."""
|
||||
worker = _make_mla_hybrid_worker(
|
||||
local_block_size=12, kernel_block_size=4, num_logical_blocks=8
|
||||
)
|
||||
@@ -1166,8 +1158,7 @@ def test_csa_linear_registration_discovers_shared_regions():
|
||||
def test_csa_linear_scratch_descs_follow_the_pages_they_overlay():
|
||||
"""The scratch ring is addressed by the regions it actually registered in,
|
||||
not by inferring that it must sit on the MLA pages: overlaying the main KV
|
||||
pages instead moves its descriptors with it.
|
||||
"""
|
||||
pages instead moves its descriptors with it."""
|
||||
worker = _make_csa_linear_ple_worker(scratch_aliases="main_kv")
|
||||
|
||||
# Same MLA regions as before, but the ring now lives in the non-MLA ones.
|
||||
@@ -1301,8 +1292,7 @@ def test_csa_linear_remote_ple_is_copied_whole():
|
||||
|
||||
def _make_ring_worker():
|
||||
"""Paged MLA group plus a per-layer ring group, no Mamba: the shape of a
|
||||
model whose sliding-window KV lives in per-request rings.
|
||||
"""
|
||||
model whose sliding-window KV lives in per-request rings."""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from vllm.config import set_current_vllm_config
|
||||
|
||||
@@ -115,8 +115,7 @@ class _BlocksMock:
|
||||
|
||||
def _stub_sw_clipping(scheduler) -> None:
|
||||
"""Make ``get_exchange_clipped_blocks`` a passthrough so tests don't
|
||||
need the full sliding-window machinery.
|
||||
"""
|
||||
need the full sliding-window machinery."""
|
||||
scheduler.get_exchange_clipped_blocks = lambda block_ids, clip_ssm=True: block_ids
|
||||
|
||||
|
||||
@@ -174,8 +173,7 @@ class TestPushScheduler:
|
||||
|
||||
def test_p_side_request_finished_stages_blocks(self):
|
||||
"""P scheduler pushes blocks into both _finished_request_blocks (lease)
|
||||
and _newly_finished_push_blocks (metadata for next step).
|
||||
"""
|
||||
and _newly_finished_push_blocks (metadata for next step)."""
|
||||
sched = make_nixl_push_scheduler()
|
||||
_stub_sw_clipping(sched)
|
||||
|
||||
@@ -194,8 +192,7 @@ class TestPushScheduler:
|
||||
|
||||
def test_build_connector_meta_drains_both_sides(self):
|
||||
"""meta.push_registrations and meta.push_finished_blocks are filled
|
||||
from the staging dicts and the staging dicts are cleared.
|
||||
"""
|
||||
from the staging dicts and the staging dicts are cleared."""
|
||||
sched = make_nixl_push_scheduler()
|
||||
_stub_sw_clipping(sched)
|
||||
|
||||
@@ -290,8 +287,7 @@ class TestPushScheduler:
|
||||
|
||||
def test_registration_watchdog_expires(self, caplog):
|
||||
"""Stale D registrations whose deadline has passed are dropped at
|
||||
``build_connector_meta`` time.
|
||||
"""
|
||||
``build_connector_meta`` time."""
|
||||
# Watchdog logs a WARNING when it drops the stale entry; that's
|
||||
# what this test is verifying, so silence it in the test report.
|
||||
caplog.set_level(
|
||||
@@ -332,8 +328,7 @@ class TestPushScheduler:
|
||||
|
||||
class _StubWriterWorker(NixlPushConnectorWorker):
|
||||
"""Construct a worker without invoking ``__init__`` so we can drive
|
||||
the matching/notif logic without bringing up NIXL or torch.
|
||||
"""
|
||||
the matching/notif logic without bringing up NIXL or torch."""
|
||||
|
||||
@classmethod
|
||||
def fresh(cls) -> _StubWriterWorker:
|
||||
@@ -439,8 +434,7 @@ class TestPushWriterRegSend:
|
||||
"""``_do_send_reg_notif``: the PUSH_REG goes to every handshaked agent of
|
||||
the P engine, and the agent map is read under ``_handshake_lock`` so the
|
||||
send can't race the handshake done-callback's insert or a concurrent
|
||||
stale-engine eviction (X1).
|
||||
"""
|
||||
stale-engine eviction (X1)."""
|
||||
|
||||
def _reg_send_worker(self) -> tuple[_StubWriterWorker, list]:
|
||||
w = _StubWriterWorker.fresh()
|
||||
@@ -485,8 +479,7 @@ class TestPushWriterRegSend:
|
||||
def test_agent_read_is_serialized_with_handshake_lock(self):
|
||||
"""While another thread holds ``_handshake_lock`` (as the handshake
|
||||
done-callback does while writing ``_remote_agents``), the writer's
|
||||
check-then-send must not run ahead of the critical section.
|
||||
"""
|
||||
check-then-send must not run ahead of the critical section."""
|
||||
w, failures = self._reg_send_worker()
|
||||
w._remote_agents["prefill-engine"] = {(0, 0): "agent-0"}
|
||||
rd = _registration_data("req-1")
|
||||
@@ -596,8 +589,7 @@ class TestPushWriterMatching:
|
||||
class TestPushWriterStartLoadKv:
|
||||
def test_finished_blocks_inbox_matches_stashed_registration(self):
|
||||
"""Run the writer-loop's finished-blocks drain against a
|
||||
pre-populated _pending_d_registrations entry.
|
||||
"""
|
||||
pre-populated _pending_d_registrations entry."""
|
||||
w = _StubWriterWorker.fresh()
|
||||
w._pending_d_registrations["req-C"] = _registration_data("req-C")
|
||||
|
||||
@@ -622,8 +614,7 @@ class TestPushWriterStartLoadKv:
|
||||
|
||||
def test_start_load_kv_enqueues_to_writer(self):
|
||||
"""``start_load_kv`` should hand registrations + finished blocks
|
||||
to the writer queues without doing matching itself.
|
||||
"""
|
||||
to the writer queues without doing matching itself."""
|
||||
w = _StubWriterWorker.fresh()
|
||||
# Stub heartbeats to a no-op; tests don't exercise the heartbeat
|
||||
# path here.
|
||||
@@ -681,8 +672,7 @@ def test_do_start_push_kv_defers_then_writes_when_handshake_ready():
|
||||
NIXL op from the writer or the executor callback); once it resolves the
|
||||
request is re-queued on ``_deferred_push_inbox`` with the wake set; and on
|
||||
re-drive with the handshake ready the WRITE is issued with a correct
|
||||
ReqMeta.
|
||||
"""
|
||||
ReqMeta."""
|
||||
w = _StubWriterWorker.fresh()
|
||||
w._logical_to_kernel_block_ids = lambda x, ratio: x
|
||||
xfer_calls: list[dict[str, Any]] = []
|
||||
@@ -722,8 +712,7 @@ def test_do_start_push_kv_defers_then_writes_when_handshake_ready():
|
||||
def test_do_start_push_kv_drops_request_on_handshake_failure():
|
||||
"""Handshake raises: the request is dropped (not re-queued, no WRITE) and
|
||||
the failure is logged. Blocks are reclaimed by the lease/watchdog, matching
|
||||
the old blocking behaviour.
|
||||
"""
|
||||
the old blocking behaviour."""
|
||||
w = _StubWriterWorker.fresh()
|
||||
w._logical_to_kernel_block_ids = lambda x, ratio: x
|
||||
xfer_calls: list[dict[str, Any]] = []
|
||||
@@ -745,8 +734,7 @@ def test_do_start_push_kv_drops_request_on_handshake_failure():
|
||||
|
||||
def test_writer_loop_drains_deferred_push_inbox():
|
||||
"""The writer loop drains ``_deferred_push_inbox`` and re-drives
|
||||
``_do_start_push_kv`` for each entry (event-driven, no polling).
|
||||
"""
|
||||
``_do_start_push_kv`` for each entry (event-driven, no polling)."""
|
||||
w = _StubWriterWorker.fresh()
|
||||
w.nixl_wrapper = MagicMock()
|
||||
w.nixl_wrapper.get_new_notifs.return_value = {}
|
||||
@@ -779,8 +767,7 @@ def test_writer_loop_drains_deferred_push_inbox():
|
||||
|
||||
def _eviction_worker(engine_ttl: float) -> NixlPushConnectorWorker:
|
||||
"""A push worker wired to drive the base ``_ensure_handshake`` eviction
|
||||
path (``_evict_stale_engines`` + ``_cleanup_remote_engine``).
|
||||
"""
|
||||
path (``_evict_stale_engines`` + ``_cleanup_remote_engine``)."""
|
||||
w = _StubWriterWorker.fresh()
|
||||
w._engine_ttl = engine_ttl
|
||||
w._engine_last_active = {}
|
||||
@@ -802,8 +789,7 @@ def _eviction_worker(engine_ttl: float) -> NixlPushConnectorWorker:
|
||||
def test_active_push_refreshes_engine_last_active():
|
||||
"""The base ``_ensure_handshake`` refreshes liveness only on a new
|
||||
handshake, so an active push to an already-connected engine must refresh
|
||||
``_engine_last_active`` itself or it is reaped mid-stream (S1).
|
||||
"""
|
||||
``_engine_last_active`` itself or it is reaped mid-stream (S1)."""
|
||||
w = _eviction_worker(engine_ttl=3600.0)
|
||||
# Already-connected D -> _ensure_handshake returns None, WRITE runs inline.
|
||||
w._remote_agents["decode-engine"] = {(0, 0): "agent-decode"}
|
||||
@@ -819,8 +805,7 @@ def test_active_push_refreshes_engine_last_active():
|
||||
def test_stale_engine_evicted_on_push():
|
||||
"""A push drives ``_ensure_handshake`` -> ``_evict_stale_engines``, so a D
|
||||
engine silent past its TTL is reaped -- no _remote_agents / NIXL agent leak
|
||||
across D scale up/down (S1).
|
||||
"""
|
||||
across D scale up/down (S1)."""
|
||||
w = _eviction_worker(engine_ttl=30.0)
|
||||
w._remote_agents["D-old"] = {(0, 0): "agent-D-old"}
|
||||
w._engine_last_active["D-old"] = time.perf_counter() - 10_000.0
|
||||
@@ -837,8 +822,7 @@ def test_stale_engine_evicted_on_push():
|
||||
def test_cleanup_remote_engine_pops_under_handshake_lock():
|
||||
"""Regression (X1): eviction must not remove ``_remote_agents`` entries
|
||||
while another thread is inside the lock's critical section (e.g. the
|
||||
push writer reading the map to send a registration).
|
||||
"""
|
||||
push writer reading the map to send a registration)."""
|
||||
w = _eviction_worker(engine_ttl=30.0)
|
||||
w._remote_agents["D-old"] = {(0, 0): "agent-D-old"}
|
||||
|
||||
@@ -863,8 +847,7 @@ def test_cleanup_remote_engine_pops_under_handshake_lock():
|
||||
class TestPushWriterNotifs:
|
||||
def test_get_new_notifs_processes_forwarded_completion_notif(self):
|
||||
"""Non-PUSH_REG notifs forwarded by the writer thread are drained
|
||||
on the engine main thread inside ``_get_new_notifs``.
|
||||
"""
|
||||
on the engine main thread inside ``_get_new_notifs``."""
|
||||
w = _StubWriterWorker.fresh()
|
||||
# Pretend the writer thread already forwarded a completion notif
|
||||
# for a request whose KV is being received.
|
||||
@@ -910,8 +893,7 @@ class TestPushWriterNotifs:
|
||||
@staticmethod
|
||||
def _pollable_worker() -> _StubWriterWorker:
|
||||
"""Stub worker with the extra base-worker state the real
|
||||
``get_transfer_results`` needs to poll ``_sending_transfers``.
|
||||
"""
|
||||
``get_transfer_results`` needs to poll ``_sending_transfers``."""
|
||||
w = _StubWriterWorker.fresh()
|
||||
w.transfer_topo = MagicMock()
|
||||
w.nixl_wrapper = MagicMock()
|
||||
@@ -953,8 +935,7 @@ class TestPushWriterNotifs:
|
||||
@pytest.mark.parametrize(("pp_size", "is_hma"), [(1, False), (2, True)])
|
||||
def test_completed_push_send_waits_for_consumer_notif(self, pp_size, is_hma):
|
||||
"""P-side sends are completed by consumer notifs / lease expiry,
|
||||
not by local WRITE-handle completion.
|
||||
"""
|
||||
not by local WRITE-handle completion."""
|
||||
w = self._pollable_worker()
|
||||
w.pp_size = pp_size
|
||||
w._is_hma_required = is_hma
|
||||
@@ -1025,8 +1006,7 @@ class TestPushSchedulerNegative:
|
||||
|
||||
def test_request_finished_unfinished_status_does_not_stage(self):
|
||||
"""If a request is still RUNNING, request_finished must not stash
|
||||
blocks for the worker (no push needed).
|
||||
"""
|
||||
blocks for the worker (no push needed)."""
|
||||
sched = make_nixl_push_scheduler()
|
||||
_stub_sw_clipping(sched)
|
||||
|
||||
@@ -1043,8 +1023,7 @@ class TestPushSchedulerNegative:
|
||||
|
||||
def test_request_finished_empty_blocks_does_not_arm_lease(self):
|
||||
"""Empty block-id groups should still complete cleanly without
|
||||
arming the lease/finished maps.
|
||||
"""
|
||||
arming the lease/finished maps."""
|
||||
sched = make_nixl_push_scheduler()
|
||||
_stub_sw_clipping(sched)
|
||||
|
||||
@@ -1059,8 +1038,7 @@ class TestPushSchedulerNegative:
|
||||
|
||||
def test_update_connector_output_unknown_request_is_noop(self):
|
||||
"""Idempotent cleanup: clearing a request that was never staged
|
||||
must not raise or mutate other state.
|
||||
"""
|
||||
must not raise or mutate other state."""
|
||||
sched = make_nixl_push_scheduler()
|
||||
_stub_sw_clipping(sched)
|
||||
|
||||
@@ -1093,8 +1071,7 @@ class TestPushWriterNegative:
|
||||
|
||||
def test_pop_matching_registration_no_match_when_base_ids_differ(self):
|
||||
"""A registration whose base id (after stripping the random suffix)
|
||||
does NOT match the lookup request_id must not be popped.
|
||||
"""
|
||||
does NOT match the lookup request_id must not be popped."""
|
||||
w = _StubWriterWorker.fresh()
|
||||
# Two unrelated requests: different base UUIDs, so stripping the
|
||||
# trailing ``-<8 hex>`` suffix still yields different base ids.
|
||||
@@ -1110,8 +1087,7 @@ class TestPushWriterNegative:
|
||||
|
||||
def test_handle_push_reg_with_non_dict_payload_is_dropped(self, caplog):
|
||||
"""msgpack-encoded non-dict payload (e.g. a list) should be
|
||||
dropped without raising.
|
||||
"""
|
||||
dropped without raising."""
|
||||
caplog.set_level(
|
||||
logging.CRITICAL,
|
||||
logger=("vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_worker"),
|
||||
@@ -1139,8 +1115,7 @@ class TestPushWriterNegative:
|
||||
|
||||
def test_handle_push_reg_idempotent_for_same_request_id(self):
|
||||
"""Receiving the same PUSH_REG twice (e.g. P retries after a
|
||||
flake) keeps the entry staged exactly once and never fires.
|
||||
"""
|
||||
flake) keeps the entry staged exactly once and never fires."""
|
||||
w = _StubWriterWorker.fresh()
|
||||
notif = PUSH_REG_NOTIF_PREFIX + msgspec.msgpack.encode(
|
||||
_registration_data("req-dup")
|
||||
@@ -1155,8 +1130,7 @@ class TestPushWriterNegative:
|
||||
"""``get_transfer_results`` must enqueue every completed request
|
||||
in ``done_sending`` so the writer can drop stale matching state.
|
||||
Unlike the happy-path test, this verifies the *cardinality*: N
|
||||
completed requests -> N evictions, in order.
|
||||
"""
|
||||
completed requests -> N evictions, in order."""
|
||||
w = _StubWriterWorker.fresh()
|
||||
with patch.object(
|
||||
NixlPushConnectorWorker.__mro__[1],
|
||||
@@ -1179,8 +1153,7 @@ class TestPushWriterNegative:
|
||||
def test_empty_transfer_results_do_not_enqueue_eviction(self):
|
||||
"""If there's nothing newly done, no eviction should be enqueued.
|
||||
The wake event IS still set because ``get_finished`` always wakes
|
||||
the writer to drain notifs.
|
||||
"""
|
||||
the writer to drain notifs."""
|
||||
w = _StubWriterWorker.fresh()
|
||||
with patch.object(
|
||||
NixlPushConnectorWorker.__mro__[1],
|
||||
@@ -1221,8 +1194,7 @@ class TestPushWriterNegative:
|
||||
|
||||
def test_get_new_notifs_unknown_request_is_logged_and_skipped(self, caplog):
|
||||
"""A completion notif for a request the worker doesn't know
|
||||
about should be logged but not crash.
|
||||
"""
|
||||
about should be logged but not crash."""
|
||||
caplog.set_level(
|
||||
logging.CRITICAL,
|
||||
logger=("vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_worker"),
|
||||
@@ -1254,8 +1226,7 @@ class TestPushWriterNegative:
|
||||
def test_get_new_notifs_extends_lease_on_heartbeat(self):
|
||||
"""``HB:`` notifs forwarded by the writer thread must extend the
|
||||
leases of tracked P-side requests on the engine main thread, and
|
||||
ignore request IDs that aren't being tracked.
|
||||
"""
|
||||
ignore request IDs that aren't being tracked."""
|
||||
w = _StubWriterWorker.fresh()
|
||||
w.transfer_topo = MagicMock()
|
||||
# _handle_heartbeat reads ``self._lease_extension`` (set in the
|
||||
@@ -1291,13 +1262,11 @@ class TestPushWriterNegative:
|
||||
|
||||
class TestPushPipelineParallel:
|
||||
"""PP-sharded producer: per-stage completion counting, pp_size plumbing,
|
||||
and remote-region slicing.
|
||||
"""
|
||||
and remote-region slicing."""
|
||||
|
||||
def test_completion_waits_for_one_notif_per_pp_stage(self):
|
||||
"""Each PP stage WRITEs its own layers and sends one notif; D must
|
||||
collect pp_size notifs before reporting the recv done.
|
||||
"""
|
||||
collect pp_size notifs before reporting the recv done."""
|
||||
w = _StubWriterWorker.fresh()
|
||||
w.transfer_topo = MagicMock()
|
||||
request_id = "req-pp-2"
|
||||
@@ -1318,8 +1287,7 @@ class TestPushPipelineParallel:
|
||||
|
||||
def test_req_meta_reads_pp_size_from_kv_transfer_params(self):
|
||||
"""D learns the producer's pp_size from kv_transfer_params (forwarded
|
||||
by the proxy) and defaults to 1 when absent.
|
||||
"""
|
||||
by the proxy) and defaults to 1 when absent."""
|
||||
metadata = NixlConnectorMetadata()
|
||||
params = {
|
||||
"remote_block_ids": ([0],),
|
||||
@@ -1342,8 +1310,7 @@ class TestPushPipelineParallel:
|
||||
this worker holds only a contiguous layer slice; add_remote_agent
|
||||
trims the remote region list to [offset : offset + num_local_regions]
|
||||
before building descriptors. We stop right after the slice via a
|
||||
sentinel on the next collaborator call.
|
||||
"""
|
||||
sentinel on the next collaborator call."""
|
||||
block_len = 4096 * 16
|
||||
w = _StubWriterWorker.fresh() # seeds writer-thread state for teardown
|
||||
w.pp_size = 2
|
||||
@@ -1385,8 +1352,7 @@ class TestPushWriterMlaReplication:
|
||||
"""MLA latent KV is replicated across D's TP ranks, so when D_TP > P_TP
|
||||
(``tp_ratio < 0``) the producer must *WRITE* the latent into every D rank
|
||||
it handshook -- not write one and merely notify the rest, which would
|
||||
leave the un-written ranks decoding against stale KV.
|
||||
"""
|
||||
leave the un-written ranks decoding against stale KV."""
|
||||
|
||||
@staticmethod
|
||||
def _mla_worker_writing_to(d_ranks):
|
||||
@@ -1611,8 +1577,7 @@ class TestPushPrefixCaching:
|
||||
|
||||
def test_partial_prefix_hit_end_trims_producer_blocks(self):
|
||||
"""D registered only its 2 uncomputed suffix blocks; P finished the
|
||||
full 5-block sequence. P must WRITE its LAST 2 blocks into D's slots.
|
||||
"""
|
||||
full 5-block sequence. P must WRITE its LAST 2 blocks into D's slots."""
|
||||
w, _ = self._worker_driving_xfer()
|
||||
reg = _registration_data("req-pc", local_block_ids=([500, 501],))
|
||||
|
||||
|
||||
@@ -171,8 +171,7 @@ def test_gpu_memory_nixl_hma(model_name, sw_size):
|
||||
@pytest.mark.parametrize("model_name", ["google/gemma-3-1b-it"])
|
||||
def test_gpu_memory_no_nixl_baseline(model_name):
|
||||
"""Same workload without NixlConnector. Comparing driver-level memory
|
||||
between this and test_gpu_memory_nixl_hma isolates UCX/NIXL impact.
|
||||
"""
|
||||
between this and test_gpu_memory_nixl_hma isolates UCX/NIXL impact."""
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.distributed.parallel_state import cleanup_dist_env_and_memory
|
||||
|
||||
|
||||
@@ -213,8 +213,7 @@ def _multi_connector_config(swa_enabled: bool = False):
|
||||
def test_nixl_wins_load_over_cpu_offload():
|
||||
"""When NixlConnector (index 0) has matched tokens from a remote prefill, it should
|
||||
win the load: Nixl metadata tracks the recv while CPU offload metadata has no load
|
||||
scheduled.
|
||||
"""
|
||||
scheduled."""
|
||||
vllm_config, kv_cache_config = _multi_connector_config()
|
||||
scheduler = create_scheduler(vllm_config, kv_cache_config=kv_cache_config)
|
||||
mc = scheduler.connector
|
||||
@@ -251,8 +250,7 @@ def test_nixl_wins_load_over_cpu_offload():
|
||||
@patch(NIXL_WRAPPER_PATCH, FakeNixlWrapper)
|
||||
def test_cpu_offload_wins_when_nixl_has_no_match():
|
||||
"""When NixlConnector returns 0 matched tokens and SimpleCPUOffloadConnector has a
|
||||
CPU cache hit, the CPU offload connector (index 1) wins the load.
|
||||
"""
|
||||
CPU cache hit, the CPU offload connector (index 1) wins the load."""
|
||||
vllm_config, kv_cache_config = _multi_connector_config()
|
||||
scheduler = create_scheduler(vllm_config, kv_cache_config=kv_cache_config)
|
||||
mc = scheduler.connector
|
||||
@@ -311,8 +309,7 @@ def test_cpu_offload_wins_when_nixl_has_no_match():
|
||||
def test_request_finished_no_async_save(swa_enabled: bool):
|
||||
"""A normal request (no P/D) produces no async save from either connector.
|
||||
MultiConnector returns (False, None) via both request_finished and
|
||||
request_finished_all_groups, and cleans up _requests_to_connector.
|
||||
"""
|
||||
request_finished_all_groups, and cleans up _requests_to_connector."""
|
||||
from vllm.v1.request import RequestStatus
|
||||
|
||||
vllm_config, kv_cache_config = _multi_connector_config(swa_enabled=swa_enabled)
|
||||
|
||||
@@ -321,8 +321,7 @@ def test_cpu_offloading(
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA")
|
||||
def test_cpu_offloading_metrics() -> None:
|
||||
"""Verify that offloading Prometheus metrics (new flat and deprecated
|
||||
labeled) are emitted after stores and loads.
|
||||
"""
|
||||
labeled) are emitted after stores and loads."""
|
||||
extra_config: dict = {
|
||||
"cpu_bytes_to_use": 500 << 20,
|
||||
"block_size": CPU_BLOCK_SIZES,
|
||||
@@ -511,8 +510,7 @@ def test_tiering_offloading() -> None:
|
||||
|
||||
def test_fs_tiering_offloading(tmp_path) -> None:
|
||||
"""Tests OffloadingConnector with TieringOffloadingSpec
|
||||
+ fs secondary tier.
|
||||
"""
|
||||
+ fs secondary tier."""
|
||||
extra_config: dict = {
|
||||
"cpu_bytes_to_use": 1 << 30,
|
||||
"block_size": CPU_BLOCK_SIZES,
|
||||
|
||||
@@ -704,8 +704,7 @@ def test_async_load_reserves_blocks_for_inflight():
|
||||
|
||||
def test_async_loads_both_admitted_when_pool_fits():
|
||||
"""Sanity: with a pool large enough, the reservation gate admits both async
|
||||
loads (it is not over-conservative).
|
||||
"""
|
||||
loads (it is not over-conservative)."""
|
||||
vllm_config = create_vllm_config()
|
||||
BLOCK_SIZE = vllm_config.cache_config.block_size
|
||||
scheduler = create_scheduler(vllm_config, num_blocks=64)
|
||||
|
||||
@@ -129,8 +129,7 @@ def _make_ring_kv_cache_config(num_blocks: int = 16) -> KVCacheConfig:
|
||||
|
||||
def test_ring_scratch_group_is_accepted_and_never_offloaded() -> None:
|
||||
"""The ring group's capacity is not a token granularity, it holds no
|
||||
hashed blocks, and it never participates in CPU hits.
|
||||
"""
|
||||
hashed blocks, and it never participates in CPU hits."""
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector="SimpleCPUOffloadConnector",
|
||||
kv_role="kv_both",
|
||||
|
||||
@@ -235,8 +235,7 @@ class TestMambaPlanSplitHandles:
|
||||
|
||||
def test_hetero_block_size_splits(self):
|
||||
"""With a block-size ratio, single-source FA sub-block descs pass
|
||||
through whole; SSM descs are unexpanded and split per source.
|
||||
"""
|
||||
through whole; SSM descs are unexpanded and split per source."""
|
||||
plan = TPMapping(
|
||||
source_ranks_per_group=((0,), (0, 1)),
|
||||
all_source_ranks=(0, 1),
|
||||
@@ -271,8 +270,7 @@ class TestMambaPlanSplitHandles:
|
||||
|
||||
def test_hetero_block_size_head_sharded_asserts(self):
|
||||
"""Head-sharded FA reads (multiple FA sources) are incompatible with
|
||||
a block-size mismatch and must fail loudly.
|
||||
"""
|
||||
a block-size mismatch and must fail loudly."""
|
||||
plan = TPMapping(
|
||||
source_ranks_per_group=((0, 1), (0, 1)),
|
||||
all_source_ranks=(0, 1),
|
||||
|
||||
@@ -15,8 +15,7 @@ from vllm.v1.kv_offload.cpu.policies.lru import LRUCachePolicy
|
||||
class _DummyCachePolicy(CachePolicy):
|
||||
"""Minimal CachePolicy for CachePolicyFactory registration tests. Loaded
|
||||
by module path, so it must be importable at module scope (mirrors
|
||||
tests/v1/kv_offload/test_factory.py's SingleArgExternalOffloadingSpec).
|
||||
"""
|
||||
tests/v1/kv_offload/test_factory.py's SingleArgExternalOffloadingSpec)."""
|
||||
|
||||
def __init__(self, cache_capacity: int) -> None:
|
||||
self.cache_capacity = cache_capacity
|
||||
@@ -55,8 +54,7 @@ class TestCachePolicyFactory:
|
||||
|
||||
def test_pre_registered_policies_can_be_imported(self):
|
||||
"""If someone moves a policy module but forgets to update
|
||||
factory.py, CI fails.
|
||||
"""
|
||||
factory.py, CI fails."""
|
||||
for name in CachePolicyFactory._registry:
|
||||
cls = CachePolicyFactory._registry[name]()
|
||||
assert issubclass(cls, CachePolicy)
|
||||
@@ -90,8 +88,7 @@ class TestCachePolicyFactory:
|
||||
register_cache_policy() call -- this is how external projects
|
||||
integrate a custom CachePolicy without forking/patching vLLM.
|
||||
Mirrors tests/v1/kv_offload/test_factory.py's
|
||||
test_dynamic_load_via_spec_module_path.
|
||||
"""
|
||||
test_dynamic_load_via_spec_module_path."""
|
||||
policy_cls = CachePolicyFactory.get_cache_policy_cls(
|
||||
"_DummyCachePolicy", "tests.v1.kv_offload.cpu.policies.test_factory"
|
||||
)
|
||||
@@ -99,8 +96,7 @@ class TestCachePolicyFactory:
|
||||
|
||||
def test_manager_resolves_policy_via_module_path(self):
|
||||
"""End-to-end: CPUOffloadingManager resolves an unregistered policy
|
||||
purely from cache_policy_module_path.
|
||||
"""
|
||||
purely from cache_policy_module_path."""
|
||||
manager = CPUOffloadingManager(
|
||||
num_chunks=4,
|
||||
cache_policy="_DummyCachePolicy",
|
||||
@@ -110,7 +106,6 @@ class TestCachePolicyFactory:
|
||||
|
||||
def test_unregistered_policy_without_module_path_raises(self):
|
||||
"""eviction_policy not in registry + no cache_policy_module_path ->
|
||||
ValueError, same shape as the OffloadingSpecFactory error path.
|
||||
"""
|
||||
ValueError, same shape as the OffloadingSpecFactory error path."""
|
||||
with pytest.raises(ValueError, match="Unknown cache policy"):
|
||||
CachePolicyFactory.get_cache_policy_cls("nonexistent", None)
|
||||
|
||||
@@ -126,8 +126,7 @@ def _canonical_handler(gpu_tensor, cpu_tensor, mapping, gpu_to_cpu):
|
||||
def test_gpu_roundtrip_assembles_canonical_page_across_ranks():
|
||||
"""Two TP2-style rank handlers must scatter into one shared canonical
|
||||
CPU page such that each rank reloads bit-exact and a whole-page (TP1)
|
||||
reader sees both shards — the cross-topology contract.
|
||||
"""
|
||||
reader sees both shards — the cross-topology contract."""
|
||||
torch.manual_seed(0)
|
||||
num_blocks = 4
|
||||
gpu_rank = [
|
||||
@@ -204,8 +203,7 @@ def _nhd_shard_mapping(tp: int, rank: int) -> CanonicalPageMapping:
|
||||
|
||||
def _head_shard(full_kv: torch.Tensor, tp: int, rank: int) -> torch.Tensor:
|
||||
"""This rank's local page rows out of the (blocks, 2, tokens, heads,
|
||||
head_bytes) ground truth.
|
||||
"""
|
||||
head_bytes) ground truth."""
|
||||
local_heads = _TOTAL_HEADS // tp
|
||||
shard = full_kv[:, :, :, rank * local_heads : (rank + 1) * local_heads, :]
|
||||
return shard.reshape(full_kv.shape[0], -1).contiguous()
|
||||
@@ -216,8 +214,7 @@ def _head_shard(full_kv: torch.Tensor, tp: int, rank: int) -> torch.Tensor:
|
||||
def test_cross_topology_roundtrip(writer_tp: int, reader_tp: int):
|
||||
"""KV written at one tp must be readable at another: writer ranks scatter
|
||||
head shards into a shared canonical region, reader ranks gather their own
|
||||
shards, and every reader must see the writers' ground-truth bytes.
|
||||
"""
|
||||
shards, and every reader must see the writers' ground-truth bytes."""
|
||||
torch.manual_seed(0)
|
||||
num_blocks = 3
|
||||
full_kv = torch.randint(
|
||||
|
||||
@@ -167,8 +167,7 @@ def _mp_race_construct_and_write(
|
||||
) -> None:
|
||||
"""Race to construct a SharedOffloadRegion, write fill_value, then wait
|
||||
for the parent's cleanup signal before tearing down. The wait gives the
|
||||
parent a window to read the raw mmap before the creator removes the file.
|
||||
"""
|
||||
parent a window to read the raw mmap before the creator removes the file."""
|
||||
try:
|
||||
region = SharedOffloadRegion(
|
||||
engine_id=engine_id,
|
||||
@@ -197,8 +196,7 @@ def _mp_barrier_construct_and_hold(
|
||||
replicated: bool = False,
|
||||
) -> None:
|
||||
"""Construct with a real cross-process barrier, write, then hold the
|
||||
mapping (no cleanup) until the parent SIGKILLs this process.
|
||||
"""
|
||||
mapping (no cleanup) until the parent SIGKILLs this process."""
|
||||
try:
|
||||
populate_calls = 0
|
||||
get_populate_write_fn = region_module._get_populate_write_fn
|
||||
@@ -252,8 +250,7 @@ def iid():
|
||||
|
||||
def test_create_next_worker_view_shape_and_stride(iid):
|
||||
"""Returned tensor must have shape (num_chunks, tensor_page_size) and
|
||||
stride (row_stride, 1) where row_stride = cpu_page_size * num_workers.
|
||||
"""
|
||||
stride (row_stride, 1) where row_stride = cpu_page_size * num_workers."""
|
||||
with _region(iid, num_chunks=4, cpu_page_size=2 * PAGE_SIZE) as r:
|
||||
t = r.create_next_worker_view(PAGE_SIZE)
|
||||
assert t.shape == (4, PAGE_SIZE)
|
||||
@@ -293,8 +290,7 @@ def test_create_next_worker_view_row_stride_with_multiple_workers(iid):
|
||||
|
||||
def test_create_next_worker_view_cursor_advances(iid):
|
||||
"""Each create_next_worker_view call must advance _worker_offset by
|
||||
tensor_page_size.
|
||||
"""
|
||||
tensor_page_size."""
|
||||
with _region(iid, cpu_page_size=3 * PAGE_SIZE) as r:
|
||||
assert r._worker_offset == 0
|
||||
r.create_next_worker_view(PAGE_SIZE)
|
||||
@@ -351,8 +347,7 @@ def test_create_next_worker_view_overflow_does_not_mutate_cursor(iid):
|
||||
|
||||
def test_create_next_worker_view_write_visible_in_raw_mmap(iid):
|
||||
"""Writes into a create_next_worker_view view must appear at the correct
|
||||
raw mmap offset
|
||||
"""
|
||||
raw mmap offset"""
|
||||
with _region(iid, num_chunks=4) as r:
|
||||
t = r.create_next_worker_view(PAGE_SIZE)
|
||||
t[2, :] = 42 # write to chunk row 2
|
||||
@@ -385,8 +380,7 @@ def test_create_next_worker_view_multi_tensor_layout(iid):
|
||||
|
||||
def test_create_next_worker_view_multiprocess_slots(iid):
|
||||
"""Each worker process calls create_next_worker_view and writes distinct data;
|
||||
the parent verifies each slot lands at the correct interleaved offset.
|
||||
"""
|
||||
the parent verifies each slot lands at the correct interleaved offset."""
|
||||
num_workers = 2
|
||||
num_chunks = 4
|
||||
|
||||
@@ -671,8 +665,7 @@ def test_multi_worker_race_shared_memory_visible(iid):
|
||||
|
||||
def test_multiprocess_race_construct_and_write(iid):
|
||||
"""N processes race to construct the same SharedOffloadRegion, each writes
|
||||
fill_value = rank+1 into their slot; parent verifies interleaved layout.
|
||||
"""
|
||||
fill_value = rank+1 into their slot; parent verifies interleaved layout."""
|
||||
num_workers = 4
|
||||
num_chunks = 3
|
||||
|
||||
@@ -798,8 +791,7 @@ def test_cleanup_unregisters_every_pinned_chunk(iid, monkeypatch):
|
||||
def test_cleanup_after_create_next_worker_view_releases_mmap(iid):
|
||||
"""cleanup() must close the mmap even after create_next_worker_view was called.
|
||||
create_next_worker_view returns a view that shares storage with _base; both must be
|
||||
released before mmap.close() can succeed.
|
||||
"""
|
||||
released before mmap.close() can succeed."""
|
||||
r = _make_region(iid)
|
||||
mmap_obj = r.mmap_obj
|
||||
|
||||
@@ -978,8 +970,7 @@ def test_ftruncate_failure_cleans_up_creator(monkeypatch):
|
||||
|
||||
def test_backing_file_unlinked_after_barrier(iid):
|
||||
"""The file must exist until the barrier releases (late joiners need the
|
||||
name) and be gone right after, so no exit path can leak it.
|
||||
"""
|
||||
name) and be gone right after, so no exit path can leak it."""
|
||||
path = f"/dev/shm/vllm_offload_{iid}.mmap"
|
||||
seen_at_barrier = []
|
||||
|
||||
@@ -1001,8 +992,7 @@ def test_backing_file_unlinked_after_barrier(iid):
|
||||
|
||||
def test_barrier_failure_unlinks_creator_and_raises(iid):
|
||||
"""A failed rendezvous must remove the creator's file and re-raise, not
|
||||
leave a stub that wedges the next start in _wait_for_file_size.
|
||||
"""
|
||||
leave a stub that wedges the next start in _wait_for_file_size."""
|
||||
path = f"/dev/shm/vllm_offload_{iid}.mmap"
|
||||
with pytest.raises(RuntimeError, match="peer died"):
|
||||
_make_region(iid, barrier=MagicMock(side_effect=RuntimeError("peer died")))
|
||||
@@ -1012,8 +1002,7 @@ def test_barrier_failure_unlinks_creator_and_raises(iid):
|
||||
@pytest.mark.parametrize("replicated", [False, True])
|
||||
def test_mp_barrier_unlinks_file_and_survives_sigkill(iid, replicated):
|
||||
"""With a real cross-process barrier every worker maps one shared inode,
|
||||
the name is gone while they run, and SIGKILL leaks nothing.
|
||||
"""
|
||||
the name is gone while they run, and SIGKILL leaks nothing."""
|
||||
num_workers = 2
|
||||
path = f"/dev/shm/vllm_offload_{iid}.mmap"
|
||||
ctx = get_mp_context()
|
||||
@@ -1052,8 +1041,7 @@ def test_mp_barrier_unlinks_file_and_survives_sigkill(iid, replicated):
|
||||
@pytest.mark.parametrize("failure_phase", ["shm", "memory", "population"])
|
||||
def test_setup_failure_before_barrier_releases_peers(iid, monkeypatch, failure_phase):
|
||||
"""A worker that dies before the rendezvous must still arrive at the
|
||||
barrier, or its peers block in the collective until it times out.
|
||||
"""
|
||||
barrier, or its peers block in the collective until it times out."""
|
||||
failure = MagicMock(side_effect=ValueError("Insufficient space"))
|
||||
memory_check = failure if failure_phase == "memory" else None
|
||||
if failure_phase == "shm":
|
||||
@@ -1084,8 +1072,7 @@ def test_setup_failure_before_barrier_releases_peers(iid, monkeypatch, failure_p
|
||||
|
||||
def test_mmap_failure_unlinks_creator_before_releasing_peers(iid, monkeypatch):
|
||||
"""A creator that fails after sizing the file must drop it before arriving
|
||||
at the barrier, so the next start does not land on a stale file.
|
||||
"""
|
||||
at the barrier, so the next start does not land on a stale file."""
|
||||
path = f"/dev/shm/vllm_offload_{iid}.mmap"
|
||||
monkeypatch.setattr("mmap.mmap", MagicMock(side_effect=OSError("mmap")))
|
||||
seen_at_barrier = []
|
||||
@@ -1098,8 +1085,7 @@ def test_mmap_failure_unlinks_creator_before_releasing_peers(iid, monkeypatch):
|
||||
|
||||
def test_barrier_release_failure_keeps_original_error(iid, monkeypatch):
|
||||
"""When releasing the peers fails too, the setup error that explains the
|
||||
failure must be the one that propagates.
|
||||
"""
|
||||
failure must be the one that propagates."""
|
||||
import vllm.v1.kv_offload.cpu.shared_offload_region as region
|
||||
|
||||
monkeypatch.setattr(
|
||||
|
||||
@@ -48,8 +48,7 @@ def _remote_prefiller_kv_params(
|
||||
kv_request_id: str = "req-1",
|
||||
) -> dict:
|
||||
"""Decoder-side kv_transfer_params: ``remote_prefiller`` sub-dict carries
|
||||
kv_request_id + remote_host + remote_port.
|
||||
"""
|
||||
kv_request_id + remote_host + remote_port."""
|
||||
return {
|
||||
"remote_prefiller": {
|
||||
"kv_request_id": kv_request_id,
|
||||
@@ -65,8 +64,7 @@ def _remote_kv_source_kv_params(
|
||||
kv_request_id: str = "req-1",
|
||||
) -> dict:
|
||||
"""Symmetric-P2P consumer kv_transfer_params: ``remote_kv_source`` sub-dict has
|
||||
the same shape as ``remote_prefiller`` (kv_request_id + remote_host + port).
|
||||
"""
|
||||
the same shape as ``remote_prefiller`` (kv_request_id + remote_host + port)."""
|
||||
return {
|
||||
"remote_kv_source": {
|
||||
"kv_request_id": kv_request_id,
|
||||
@@ -78,8 +76,7 @@ def _remote_kv_source_kv_params(
|
||||
|
||||
def _remote_decoder_kv_params(kv_request_id: str = "req-1") -> dict:
|
||||
"""Prefiller-side kv_transfer_params: ``remote_decoder`` sub-dict carries
|
||||
kv_request_id only.
|
||||
"""
|
||||
kv_request_id only."""
|
||||
return {"remote_decoder": {"kv_request_id": kv_request_id}}
|
||||
|
||||
|
||||
@@ -244,8 +241,7 @@ class TestLookup:
|
||||
def test_lookup_returns_miss_without_prefill_key(self):
|
||||
"""No ``remote_prefiller`` sub-dict means the request was not routed for
|
||||
remote prefill — local prefill should run instead, so lookup()
|
||||
returns MISS even when a stale ``remote_decoder`` block is present.
|
||||
"""
|
||||
returns MISS even when a stale ``remote_decoder`` block is present."""
|
||||
mgr = _make_manager()
|
||||
ctx = _req_context(kv_params=_remote_decoder_kv_params())
|
||||
assert mgr.lookup(b"key", ctx) is LookupResult.MISS
|
||||
@@ -313,8 +309,7 @@ class TestServeExternalRequests:
|
||||
def test_flushes_failed_serve_ctxs_then_serves_each_session(self):
|
||||
"""serve_external_requests releases the failed serves left by
|
||||
reaped sessions via parent.on_request_finished (clearing the
|
||||
queue), then delegates to every live session with the same parent.
|
||||
"""
|
||||
queue), then delegates to every live session with the same parent."""
|
||||
mgr = _make_manager()
|
||||
ctx = ReqContext(req_id="p2p:peer:req-1:lu1")
|
||||
mgr._failed_serve_ctxs = [ctx]
|
||||
@@ -368,8 +363,7 @@ class TestSubmitStore:
|
||||
def test_no_binding_yet_parks_in_unbound_stores(self):
|
||||
"""submit_store without a bound session buffers the batch keyed
|
||||
by kv_request_id; no session is pre-created (peer_id is unknown
|
||||
to the producer at store time).
|
||||
"""
|
||||
to the producer at store time)."""
|
||||
mgr = _make_manager()
|
||||
job = _job_metadata(
|
||||
job_id=1,
|
||||
@@ -396,8 +390,7 @@ class TestSubmitStore:
|
||||
def test_routes_to_bound_session(self):
|
||||
"""If a session has already received FetchMsg for this
|
||||
kv_request_id (so _kv_to_session is populated), submit_store
|
||||
forwards directly to that session rather than re-buffering.
|
||||
"""
|
||||
forwards directly to that session rather than re-buffering."""
|
||||
mgr = _make_manager()
|
||||
bound = _FakeSession(peer_id="10.0.0.1:8000", connected=True)
|
||||
mgr._kv_to_session["req-1"] = bound # type: ignore[assignment]
|
||||
@@ -420,8 +413,7 @@ class TestSubmitStore:
|
||||
def test_extra_top_level_keys_are_ignored(self):
|
||||
"""Producer-side kv_transfer_params should not pre-create a
|
||||
session even when a stale caller still passes a top-level
|
||||
``remote_host``/``remote_port`` next to ``remote_decoder``.
|
||||
"""
|
||||
``remote_host``/``remote_port`` next to ``remote_decoder``."""
|
||||
mgr = _make_manager()
|
||||
params = _remote_decoder_kv_params()
|
||||
params["remote_host"] = "stale"
|
||||
@@ -465,8 +457,7 @@ class TestSubmitLoad:
|
||||
|
||||
def test_happy_path_with_active_session(self):
|
||||
"""When the peer's session exists, submit_load forwards to
|
||||
session.request_blocks and does NOT add a finished result yet.
|
||||
"""
|
||||
session.request_blocks and does NOT add a finished result yet."""
|
||||
mgr = _make_manager()
|
||||
peer_id = "10.0.0.1:8000"
|
||||
existing = _FakeSession(peer_id=peer_id, connected=True)
|
||||
@@ -486,8 +477,7 @@ class TestSubmitLoad:
|
||||
def test_missing_consumer_flag_fails(self):
|
||||
"""Peer fields present but neither do_remote_prefill nor
|
||||
do_p2p_fetch is set — submit_load fails the job rather than
|
||||
emit a stray FetchMsg.
|
||||
"""
|
||||
emit a stray FetchMsg."""
|
||||
mgr = _make_manager()
|
||||
params = {
|
||||
"remote_host": "10.0.0.1",
|
||||
@@ -531,8 +521,7 @@ class TestOnRequestFinished:
|
||||
def test_decoder_side_calls_session_finish_request(self):
|
||||
"""Decoder-side finish (``remote_prefiller`` set) still routes via peer_id
|
||||
because the consumer addresses the producer it loaded from. The
|
||||
session's finish_request cancels the client-role load.
|
||||
"""
|
||||
session's finish_request cancels the client-role load."""
|
||||
mgr = _make_manager()
|
||||
peer_id = "10.0.0.1:8000"
|
||||
session = _FakeSession(peer_id=peer_id)
|
||||
@@ -544,8 +533,7 @@ class TestOnRequestFinished:
|
||||
def test_p2p_consumer_side_calls_session_finish_request(self):
|
||||
"""Symmetric-P2P consumer finish (``remote_kv_source`` set) routes via peer_id
|
||||
so the session drops any pending lookups (cancel_lookups) and
|
||||
cancels any inbound load.
|
||||
"""
|
||||
cancels any inbound load."""
|
||||
mgr = _make_manager()
|
||||
peer_id = "10.0.0.1:8000"
|
||||
session = _FakeSession(peer_id=peer_id)
|
||||
@@ -556,8 +544,7 @@ class TestOnRequestFinished:
|
||||
|
||||
def test_prefiller_bound_id_routes_via_kv_to_session(self):
|
||||
"""Prefiller-side finish for an id whose session is already bound
|
||||
(FetchMsg received) routes via _kv_to_session and pops the entry.
|
||||
"""
|
||||
(FetchMsg received) routes via _kv_to_session and pops the entry."""
|
||||
mgr = _make_manager()
|
||||
bound = _FakeSession(peer_id="some-peer:1", connected=True)
|
||||
mgr._kv_to_session["req-1"] = bound # type: ignore[assignment]
|
||||
@@ -571,8 +558,7 @@ class TestOnRequestFinished:
|
||||
and no session binding is a no-op on `_unbound_stores`. The
|
||||
parked batches survive until a peer fetches them or the
|
||||
`_reap_unbound_stores` timeout fires — `on_request_finished`
|
||||
must not evict them.
|
||||
"""
|
||||
must not evict them."""
|
||||
from vllm.v1.kv_offload.tiering.p2p.manager import _UnboundStoreBatch
|
||||
|
||||
mgr = _make_manager()
|
||||
@@ -742,8 +728,7 @@ class TestGetFinished:
|
||||
|
||||
def test_reap_fails_probes(self):
|
||||
"""A reaped session's in-flight lookups land in _failed_req_ids so
|
||||
the consumer's lookup() returns MISS instead of RETRY forever.
|
||||
"""
|
||||
the consumer's lookup() returns MISS instead of RETRY forever."""
|
||||
|
||||
class FakeData:
|
||||
def remove_remote_peer(self, pid):
|
||||
@@ -778,8 +763,7 @@ class TestGetFinished:
|
||||
def test_unbound_store_reaped_after_timeout(self):
|
||||
"""Unbound stores past the reap deadline surface as failed and their
|
||||
kv_request_id lands in _reaped_stores so a late FetchMsg is rejected
|
||||
instead of parking demand nothing can satisfy.
|
||||
"""
|
||||
instead of parking demand nothing can satisfy."""
|
||||
from vllm.v1.kv_offload.tiering.p2p.manager import _UnboundStoreBatch
|
||||
|
||||
mgr = self._make()
|
||||
@@ -801,8 +785,7 @@ class TestGetFinished:
|
||||
|
||||
def test_submit_store_parks_unbound_batch(self):
|
||||
"""submit_store on an unbound id appends a batch with a fresh
|
||||
submitted_at stamp so the unbound-store sweep can age it out.
|
||||
"""
|
||||
submitted_at stamp so the unbound-store sweep can age it out."""
|
||||
mgr = _make_manager()
|
||||
job = _job_metadata(
|
||||
job_id=1, kv_params=_remote_decoder_kv_params(kv_request_id="req-1")
|
||||
@@ -824,21 +807,18 @@ class TestHasPendingWork:
|
||||
"""has_pending_work() must always return True so the engine keeps
|
||||
ticking the offload pipeline — that's the only thread driving
|
||||
_control.poll() (incoming peer connects) and session.poll()
|
||||
(incoming fetch messages on existing sessions).
|
||||
"""
|
||||
(incoming fetch messages on existing sessions)."""
|
||||
|
||||
def test_returns_true_unconditionally_to_keep_engine_ticking(self):
|
||||
"""Even with no sessions and no jobs, has_pending_work() returns
|
||||
True so the engine keeps calling get_finished_jobs(), which is
|
||||
what drives _control.poll() for inbound peer connects.
|
||||
"""
|
||||
what drives _control.poll() for inbound peer connects."""
|
||||
mgr = _make_manager()
|
||||
assert mgr.has_pending_work() is True
|
||||
|
||||
def test_returns_true_even_when_sessions_present(self):
|
||||
"""The result is the same regardless of session state — there is
|
||||
no 'idle' branch.
|
||||
"""
|
||||
no 'idle' branch."""
|
||||
mgr = _make_manager()
|
||||
mgr._sessions["peer:1"] = _FakeSession(peer_id="peer:1") # type: ignore[assignment]
|
||||
assert mgr.has_pending_work() is True
|
||||
@@ -851,8 +831,7 @@ class TestHasPendingWork:
|
||||
|
||||
class _ShutdownFakeData:
|
||||
"""Fake DataTransport that records cancel/poll/close calls and
|
||||
drives the wait-cancel loop with a scriptable `still` queue.
|
||||
"""
|
||||
drives the wait-cancel loop with a scriptable `still` queue."""
|
||||
|
||||
def __init__(self, still_queue: list[list[int]] | None = None) -> None:
|
||||
# Each list in still_queue is the set of ids the next
|
||||
@@ -896,8 +875,7 @@ class _ShutdownFakeControl:
|
||||
class TestShutdownDrain:
|
||||
"""shutdown() drains inflight transfers via cancel(mode='wait')
|
||||
before calling _data.close(), with a 3s deadline fallback to
|
||||
cancel(mode='immediate').
|
||||
"""
|
||||
cancel(mode='immediate')."""
|
||||
|
||||
def _prep(
|
||||
self,
|
||||
@@ -1316,8 +1294,7 @@ class _RecordingConn:
|
||||
|
||||
class TestAcceptNewPeers:
|
||||
"""A second inbound from an already-connected peer is rejected and the
|
||||
new conn is closed; the existing session is left untouched.
|
||||
"""
|
||||
new conn is closed; the existing session is left untouched."""
|
||||
|
||||
def test_duplicate_connection_is_closed_and_existing_session_untouched(self):
|
||||
mgr = _make_manager()
|
||||
@@ -1339,8 +1316,7 @@ class TestAcceptNewPeers:
|
||||
"""An inbound conn from a peer with no existing session creates
|
||||
a fresh connected session and registers it under conn.peer_id.
|
||||
The prefiller has no pre-created pending session anymore — the
|
||||
first signal of a peer's existence is its inbound connection.
|
||||
"""
|
||||
first signal of a peer's existence is its inbound connection."""
|
||||
|
||||
class FakeData:
|
||||
block_len = 4096
|
||||
@@ -1387,8 +1363,7 @@ class TestAcceptNewPeers:
|
||||
|
||||
class TestPollOnce:
|
||||
"""_poll_once must drain control, accept new peers, poll every session,
|
||||
surface results, and reap dead sessions — in that order.
|
||||
"""
|
||||
surface results, and reap dead sessions — in that order."""
|
||||
|
||||
def test_orchestrates_accept_poll_and_reap(self):
|
||||
mgr = _make_manager()
|
||||
@@ -1444,8 +1419,7 @@ class TestPollOnce:
|
||||
def test_new_fetch_id_binds_and_replays_unbound_batches(self):
|
||||
"""When session.poll() reports a kv_request_id whose FetchMsg
|
||||
arrived this tick, the manager binds it to that session and
|
||||
replays every parked submit_store batch via add_stored_blocks.
|
||||
"""
|
||||
replays every parked submit_store batch via add_stored_blocks."""
|
||||
from vllm.v1.kv_offload.tiering.p2p.manager import _UnboundStoreBatch
|
||||
|
||||
mgr = _make_manager()
|
||||
@@ -1479,8 +1453,7 @@ class TestPollOnce:
|
||||
|
||||
def test_new_fetch_id_with_no_unbound_still_binds(self):
|
||||
"""A FetchMsg for a kv_request_id with no parked batches still
|
||||
records the binding so subsequent submit_stores route fast.
|
||||
"""
|
||||
records the binding so subsequent submit_stores route fast."""
|
||||
mgr = _make_manager()
|
||||
peer = "10.0.0.1:8000"
|
||||
sess = _FakeSession(
|
||||
@@ -1503,8 +1476,7 @@ class TestPollOnce:
|
||||
|
||||
def test_failed_load_records_kv_request_id(self):
|
||||
"""A LoadResult(success=False) from session.poll() must add its
|
||||
kv_request_id to _failed_req_ids so future lookups return MISS.
|
||||
"""
|
||||
kv_request_id to _failed_req_ids so future lookups return MISS."""
|
||||
mgr = _make_manager()
|
||||
peer = "10.0.0.1:8000"
|
||||
sess = _FakeSession(
|
||||
@@ -1568,8 +1540,7 @@ class TestDrainJobs:
|
||||
|
||||
def test_logs_warning_after_5s_then_completes(self, monkeypatch):
|
||||
"""A session that stays inflight past 5s triggers the warning, and
|
||||
once it clears the loop returns.
|
||||
"""
|
||||
once it clears the loop returns."""
|
||||
mgr = _make_manager()
|
||||
mgr._control = _DrainCtrl() # type: ignore[assignment]
|
||||
sess = _FakeSession(peer_id="peer:1")
|
||||
@@ -1627,8 +1598,7 @@ class TestDrainJobs:
|
||||
class TestOnScheduleEnd:
|
||||
def test_is_noop(self):
|
||||
"""on_schedule_end is a documented no-op; just confirm it doesn't
|
||||
raise and doesn't mutate state.
|
||||
"""
|
||||
raise and doesn't mutate state."""
|
||||
mgr = _make_manager()
|
||||
before_sessions = dict(mgr._sessions)
|
||||
before_jobs = list(mgr._finished_jobs)
|
||||
@@ -1654,8 +1624,7 @@ class TestConnectionDeathMidTransfer:
|
||||
prefiller-side store no longer travels through the session at store
|
||||
time (it's parked in _unbound_stores keyed by kv_request_id), so its
|
||||
cleanup on connection death is via on_request_finished or the
|
||||
unbound-store timeout — covered separately below.
|
||||
"""
|
||||
unbound-store timeout — covered separately below."""
|
||||
|
||||
def test_dead_connection_with_pending_work_surfaces_failures(self):
|
||||
mgr_a, mgr_b = _build_paired_managers()
|
||||
|
||||
@@ -626,8 +626,7 @@ class TestClientFlows:
|
||||
"""collect_results / has_active_loads use the _active_loads work-list,
|
||||
armed when a fetch is issued and discarded exactly when its load
|
||||
clears — a probe-only request never enters it, and completion empties
|
||||
it while the entry may briefly linger for GC.
|
||||
"""
|
||||
it while the entry may briefly linger for GC."""
|
||||
session, conn, _ = _make_session()
|
||||
_activate(session, conn)
|
||||
client = session._client
|
||||
@@ -704,8 +703,7 @@ class TestClientFlows:
|
||||
def test_load_abort_ack_clears_request(self):
|
||||
"""After load timeout sends AbortFetch, an arriving AbortAckMsg from
|
||||
the peer surfaces the failure cleanly and removes the request from
|
||||
_requests — covers the on_abort_ack arrival path.
|
||||
"""
|
||||
_requests — covers the on_abort_ack arrival path."""
|
||||
session, conn, _ = _make_session()
|
||||
_activate(session, conn)
|
||||
session.request_blocks(
|
||||
@@ -742,8 +740,7 @@ class TestLookupFlow:
|
||||
def test_aggregate_flush_resolve_round_trip(self):
|
||||
"""register_lookup → flush sends one LookupMsg → response
|
||||
resolves entries → register_lookup returns the cached bool on
|
||||
every call, and repeat probes never re-issue a LookupMsg.
|
||||
"""
|
||||
every call, and repeat probes never re-issue a LookupMsg."""
|
||||
session, conn, _ = _make_session()
|
||||
_activate(session, conn)
|
||||
|
||||
@@ -796,8 +793,7 @@ class TestLookupFlow:
|
||||
def test_request_blocks_clears_probe_cache(self):
|
||||
"""A resolved HIT probe is popped when its fetch is issued, so a
|
||||
re-scheduled request re-probes instead of trusting the stale True
|
||||
(the served block is unpinned and may have been evicted).
|
||||
"""
|
||||
(the served block is unpinned and may have been evicted)."""
|
||||
session, conn, _ = _make_session()
|
||||
_activate(session, conn)
|
||||
|
||||
@@ -883,8 +879,7 @@ class TestLookupFlow:
|
||||
def test_multiple_lookup_msgs_across_steps(self):
|
||||
"""A request's block set may be discovered across scheduler steps:
|
||||
each step that registers new keys flushes its own LookupMsg for
|
||||
the same kv_request_id, carrying only the newly-probed keys.
|
||||
"""
|
||||
the same kv_request_id, carrying only the newly-probed keys."""
|
||||
session, conn, _ = _make_session()
|
||||
_activate(session, conn)
|
||||
|
||||
@@ -912,8 +907,7 @@ class TestLookupFlow:
|
||||
|
||||
def test_split_response_resolves_across_messages(self):
|
||||
"""Producer may answer one LookupMsg's keys across multiple
|
||||
LookupRespMsgs — pairs are self-describing so each lands.
|
||||
"""
|
||||
LookupRespMsgs — pairs are self-describing so each lands."""
|
||||
session, conn, _ = _make_session()
|
||||
_activate(session, conn)
|
||||
|
||||
@@ -960,8 +954,7 @@ class TestLookupFlow:
|
||||
def test_finish_after_flushed_lookup_sends_empty_fetch(self):
|
||||
"""LookupMsg flushed but no FetchMsg sent (all-miss case) →
|
||||
finish_request emits an empty FetchMsg so the peer can drop its
|
||||
lookup state and call parent.on_request_finished.
|
||||
"""
|
||||
lookup state and call parent.on_request_finished."""
|
||||
session, conn, _ = _make_session()
|
||||
_activate(session, conn)
|
||||
|
||||
@@ -979,8 +972,7 @@ class TestLookupFlow:
|
||||
|
||||
def test_finish_without_flushed_lookup_sends_no_fetch(self):
|
||||
"""No LookupMsg was ever sent → finish_request must not emit an
|
||||
empty FetchMsg (the peer has no state to release).
|
||||
"""
|
||||
empty FetchMsg (the peer has no state to release)."""
|
||||
session, conn, _ = _make_session()
|
||||
_activate(session, conn)
|
||||
|
||||
@@ -995,8 +987,7 @@ class TestLookupFlow:
|
||||
|
||||
def test_finish_after_real_fetch_sends_no_second_fetch(self):
|
||||
"""A real FetchMsg was already sent for the id → finish_request
|
||||
must not emit a second (empty) FetchMsg.
|
||||
"""
|
||||
must not emit a second (empty) FetchMsg."""
|
||||
session, conn, _ = _make_session()
|
||||
_activate(session, conn)
|
||||
|
||||
@@ -1027,8 +1018,7 @@ class TestLookupFlow:
|
||||
"""``poll()`` only enqueues an inbound LookupMsg — no response is
|
||||
sent until ``serve_external_requests``. With an all-miss parent
|
||||
the aggregated LookupRespMsg carries the same keys and
|
||||
``hits=[False, ...]``.
|
||||
"""
|
||||
``hits=[False, ...]``."""
|
||||
session, conn, _ = _make_session()
|
||||
_activate(session, conn)
|
||||
|
||||
@@ -1088,8 +1078,7 @@ class TestServerLookupHandling:
|
||||
def test_immediate_hits_create_one_store_job(self):
|
||||
"""All-HIT batch: one create_store_job call with all keys, one
|
||||
LookupRespMsg with hits=[True]*N, on_request_finished fires at the
|
||||
end of serve, and `available` is populated for the eventual fetch.
|
||||
"""
|
||||
end of serve, and `available` is populated for the eventual fetch."""
|
||||
cb = FakeParent(stored={b"hA": 1, b"hB": 2, b"hC": 3})
|
||||
session, conn, _ = _make_session()
|
||||
_activate(session, conn)
|
||||
@@ -1119,8 +1108,7 @@ class TestServerLookupHandling:
|
||||
|
||||
def test_all_misses_no_store_job_finish_fires(self):
|
||||
"""All-MISS batch: no create_store_job call; one LookupRespMsg
|
||||
with hits=[False]*N; on_request_finished fires at end of serve.
|
||||
"""
|
||||
with hits=[False]*N; on_request_finished fires at end of serve."""
|
||||
cb = FakeParent()
|
||||
session, conn, _ = _make_session()
|
||||
_activate(session, conn)
|
||||
@@ -1140,8 +1128,7 @@ class TestServerLookupHandling:
|
||||
"""HIT/MISS resolutions do not go out on first sight when any
|
||||
key is still HIT_PENDING / RETRY. The lookup parks until every
|
||||
key has settled (or the deadline fires), then one
|
||||
LookupRespMsg carries all keys in wire order.
|
||||
"""
|
||||
LookupRespMsg carries all keys in wire order."""
|
||||
cb = FakeParent(
|
||||
stored={b"hA": 1},
|
||||
pending={b"hB"},
|
||||
@@ -1170,8 +1157,7 @@ class TestServerLookupHandling:
|
||||
the deferred aggregate response: one LookupRespMsg carrying
|
||||
both keys in wire order, and one create_store_job call per
|
||||
HIT (the second HIT is pinned when it resolves, not when the
|
||||
response goes out).
|
||||
"""
|
||||
response goes out)."""
|
||||
cb = FakeParent(stored={b"hA": 1}, pending={b"hB"})
|
||||
session, conn, _ = _make_session()
|
||||
_activate(session, conn)
|
||||
@@ -1207,8 +1193,7 @@ class TestServerLookupHandling:
|
||||
def test_pending_timeout_replies_miss_no_store_job(self):
|
||||
"""A HIT_PENDING key that stays pending past the batch
|
||||
``deadline`` is force-MISS and never pinned; the deferred
|
||||
aggregate response fires with hits=[False].
|
||||
"""
|
||||
aggregate response fires with hits=[False]."""
|
||||
cb = FakeParent(pending={b"hA"})
|
||||
session, conn, _ = _make_session()
|
||||
_activate(session, conn)
|
||||
@@ -1235,8 +1220,7 @@ class TestServerLookupHandling:
|
||||
|
||||
def test_finish_request_called_per_lookup_msg_not_per_kv_request_id(self):
|
||||
"""Two LookupMsgs for the same kv_request_id get distinct ctxs
|
||||
and two on_request_finished calls (one per batch).
|
||||
"""
|
||||
and two on_request_finished calls (one per batch)."""
|
||||
cb = FakeParent(stored={b"hA": 1, b"hB": 2})
|
||||
session, conn, _ = _make_session()
|
||||
_activate(session, conn)
|
||||
@@ -1260,8 +1244,7 @@ class TestServerLookupHandling:
|
||||
"""Tearing the session down with a parked batch returns the
|
||||
synthetic ctx as a failed serve (no parent handle at teardown) so
|
||||
the manager can release the TieringManager's state on its next
|
||||
serve.
|
||||
"""
|
||||
serve."""
|
||||
cb = FakeParent(pending={b"hA"})
|
||||
session, conn, _ = _make_session()
|
||||
_activate(session, conn)
|
||||
@@ -1282,8 +1265,7 @@ class TestServerLookupHandling:
|
||||
def test_wire_finish_drops_pending_batches_for_kv_request_id(self):
|
||||
"""``ServerRole.finish(kv_request_id)`` drops every parked batch
|
||||
whose kv_request_id matches and queues its ctx for the next
|
||||
serve's on_request_finished.
|
||||
"""
|
||||
serve's on_request_finished."""
|
||||
cb = FakeParent(pending={b"hA", b"hB"})
|
||||
session, conn, _ = _make_session()
|
||||
_activate(session, conn)
|
||||
@@ -1314,8 +1296,7 @@ class TestServerLookupHandling:
|
||||
def test_incoming_fetch_drops_pending_lookups_for_kv_request_id(self):
|
||||
"""A peer FetchMsg terminates the lookup phase for its id: parked
|
||||
lookups with matching kv_request_id are dropped and their
|
||||
ctx queued for on_request_finished; other kv_request_ids untouched.
|
||||
"""
|
||||
ctx queued for on_request_finished; other kv_request_ids untouched."""
|
||||
cb = FakeParent(pending={b"hA", b"hB"})
|
||||
session, conn, _ = _make_session()
|
||||
_activate(session, conn)
|
||||
@@ -1357,8 +1338,7 @@ class TestServerLookupHandling:
|
||||
def test_lookup_then_fetch_round_trip_emits_store_result(self):
|
||||
"""End-to-end: lookup pins primary slots → fetch matches them →
|
||||
NIXL transfer completes → StoreResult surfaces with the
|
||||
create_store_job's job_id (the engine releases the pin).
|
||||
"""
|
||||
create_store_job's job_id (the engine releases the pin)."""
|
||||
cb = FakeParent(stored={b"hA": 7, b"hB": 8})
|
||||
session, conn, transport = _make_session()
|
||||
_activate(session, conn)
|
||||
@@ -1480,8 +1460,7 @@ class TestServerFlows:
|
||||
|
||||
def test_abort_fetch_defers_ack_when_cancel_pending(self):
|
||||
"""If cancel(mode='wait') reports still-inflight tids, the ack is
|
||||
deferred and the abort is parked (abort_started_at set).
|
||||
"""
|
||||
deferred and the abort is parked (abort_started_at set)."""
|
||||
session, conn, transport = _make_session()
|
||||
_activate(session, conn)
|
||||
# Seed an inflight transfer for req-1 that the transport pretends
|
||||
@@ -1516,8 +1495,7 @@ class TestServerFlows:
|
||||
|
||||
def test_abort_fetch_acks_after_drain(self):
|
||||
"""Once the transport reports the tid as DONE the parked abort
|
||||
completes and AbortAckMsg is sent.
|
||||
"""
|
||||
completes and AbortAckMsg is sent."""
|
||||
session, conn, transport = _make_session()
|
||||
_activate(session, conn)
|
||||
tid = 42
|
||||
@@ -1556,8 +1534,7 @@ class TestServerFlows:
|
||||
|
||||
def test_abort_fetch_force_cancels_after_timeout(self):
|
||||
"""If wait-mode never drains, the deadline forces immediate
|
||||
cancel and an ack is still sent.
|
||||
"""
|
||||
cancel and an ack is still sent."""
|
||||
session, conn, transport = _make_session()
|
||||
_activate(session, conn)
|
||||
tid = 42
|
||||
@@ -1599,8 +1576,7 @@ class TestServerFlows:
|
||||
|
||||
def test_abort_fetch_idempotent_while_draining(self):
|
||||
"""Receiving AbortFetchMsg twice for the same kv_request_id
|
||||
keeps a single pending entry and produces a single ack.
|
||||
"""
|
||||
keeps a single pending entry and produces a single ack."""
|
||||
session, conn, transport = _make_session()
|
||||
_activate(session, conn)
|
||||
tid = 42
|
||||
@@ -1658,8 +1634,7 @@ class TestServerFlows:
|
||||
def test_store_timeout_then_late_completion_no_duplicate(self):
|
||||
"""A job timed out by _timeout_pending_store_jobs must not also
|
||||
emit a contradictory StoreResult(success=True) when the transport
|
||||
later reports the same transfer as done.
|
||||
"""
|
||||
later reports the same transfer as done."""
|
||||
session, conn, transport = _make_session()
|
||||
_activate(session, conn)
|
||||
session.add_stored_blocks("req-1", [b"k1"], [0], job_id=1)
|
||||
@@ -1692,8 +1667,7 @@ class TestServerFlows:
|
||||
def test_store_timeout_then_late_failure_no_duplicate(self):
|
||||
"""Symmetric guard: a timed-out job must not also emit a second
|
||||
StoreResult(success=False) when the transport later reports the
|
||||
same transfer as failed.
|
||||
"""
|
||||
same transfer as failed."""
|
||||
session, conn, transport = _make_session()
|
||||
_activate(session, conn)
|
||||
session.add_stored_blocks("req-1", [b"k1"], [0], job_id=1)
|
||||
@@ -1736,8 +1710,7 @@ class TestFinishRequestServerSide:
|
||||
|
||||
def test_no_inflight_unmatched_demand_sends_failure(self):
|
||||
"""finish_request with unmatched demand and no inflight ->
|
||||
immediate TransferDoneMsg(success=False); _outbound cleared.
|
||||
"""
|
||||
immediate TransferDoneMsg(success=False); _outbound cleared."""
|
||||
session, conn, _ = _make_session()
|
||||
_activate(session, conn)
|
||||
# Decoder demanded a block we never stored.
|
||||
@@ -1763,8 +1736,7 @@ class TestFinishRequestServerSide:
|
||||
|
||||
def test_with_inflight_defers_then_fires_on_last_transfer(self):
|
||||
"""finish_request with inflight defers; last transfer fires the
|
||||
early-fail message and clears _outbound.
|
||||
"""
|
||||
early-fail message and clears _outbound."""
|
||||
session, conn, transport = _make_session()
|
||||
_activate(session, conn)
|
||||
# Demand 2 blocks; we store 1 (kicks one inflight transfer).
|
||||
@@ -1801,8 +1773,7 @@ class TestFinishRequestServerSide:
|
||||
|
||||
def test_full_demand_satisfied_still_sends_success(self):
|
||||
"""finish_request must not override a fully-satisfied transfer:
|
||||
when remaining hits 0, success=True still fires.
|
||||
"""
|
||||
when remaining hits 0, success=True still fires."""
|
||||
session, conn, transport = _make_session()
|
||||
_activate(session, conn)
|
||||
conn.enqueue(
|
||||
@@ -1831,8 +1802,7 @@ class TestFinishRequestServerSide:
|
||||
"""Prefiller-first: finish_request runs before the decoder's
|
||||
fetch arrives. State is held until fetch, then
|
||||
finalized — success=True if all demand was matched against
|
||||
available blocks, else success=False.
|
||||
"""
|
||||
available blocks, else success=False."""
|
||||
# Case A: all demand satisfied by available blocks.
|
||||
session, conn, transport = _make_session()
|
||||
_activate(session, conn)
|
||||
@@ -1893,8 +1863,7 @@ class TestFinishRequestServerSide:
|
||||
"""finish_request with a stored-but-unmatched job and no inflight ->
|
||||
TransferDoneMsg(success=False) AND deferred
|
||||
StoreResult(success=False) for the submit_store'd job, instead of
|
||||
the 30s _STORE_TIMEOUT_S path.
|
||||
"""
|
||||
the 30s _STORE_TIMEOUT_S path."""
|
||||
session, conn, _ = _make_session()
|
||||
_activate(session, conn)
|
||||
# Decoder demanded b"demand"; we never stored it.
|
||||
@@ -1930,8 +1899,7 @@ class TestFinishRequestServerSide:
|
||||
def test_finish_request_remaining_zero_emits_success_via_inflight(self):
|
||||
"""Deferred-via-inflight path: finish_request with inflight, last
|
||||
transfer drains remaining to 0 -> TransferDoneMsg(success=True)
|
||||
AND StoreResult(success=True).
|
||||
"""
|
||||
AND StoreResult(success=True)."""
|
||||
session, conn, transport = _make_session()
|
||||
_activate(session, conn)
|
||||
conn.enqueue(
|
||||
@@ -2279,8 +2247,7 @@ class TestDisconnect:
|
||||
def test_send_failure_marks_connection_dead(self):
|
||||
"""A raising send must mark the connection dead, not silently drop
|
||||
the message — otherwise the session lingers alive, is never reaped,
|
||||
and in-flight lookups/loads toward the dead peer hang forever.
|
||||
"""
|
||||
and in-flight lookups/loads toward the dead peer hang forever."""
|
||||
session, conn, _ = _make_session()
|
||||
_activate(session, conn)
|
||||
assert session.alive
|
||||
@@ -2293,8 +2260,7 @@ class TestDisconnect:
|
||||
|
||||
def test_close_surfaces_inflight_lookups(self):
|
||||
"""close() reports kv_request_ids whose symmetric-P2P probe is still
|
||||
unresolved; resolved probes are not reported (their answer is in).
|
||||
"""
|
||||
unresolved; resolved probes are not reported (their answer is in)."""
|
||||
session, conn, _ = _make_session()
|
||||
_activate(session, conn)
|
||||
|
||||
@@ -2402,8 +2368,7 @@ class TestDispatchErrorHandling:
|
||||
|
||||
def test_value_error_disconnects_on_first_occurrence(self):
|
||||
"""A FetchMsg that fails validate() raises ValueError and must
|
||||
terminate the session immediately, with a DisconnectMsg sent.
|
||||
"""
|
||||
terminate the session immediately, with a DisconnectMsg sent."""
|
||||
session, conn, _ = _make_session()
|
||||
_activate(session, conn)
|
||||
# length mismatch → FetchMsg.validate raises ValueError
|
||||
@@ -2430,8 +2395,7 @@ class TestDispatchErrorHandling:
|
||||
|
||||
def test_internal_error_does_not_disconnect_once(self):
|
||||
"""A non-ValueError raised by a handler is treated as an internal
|
||||
bug: counter increments, session stays alive on a single hit.
|
||||
"""
|
||||
bug: counter increments, session stays alive on a single hit."""
|
||||
session, conn, _ = _make_session()
|
||||
_activate(session, conn)
|
||||
|
||||
@@ -2454,8 +2418,7 @@ class TestDispatchErrorHandling:
|
||||
|
||||
def test_internal_error_threshold_disconnects(self):
|
||||
"""Once consecutive non-protocol errors hit the threshold, the
|
||||
session tears down via _protocol_error.
|
||||
"""
|
||||
session tears down via _protocol_error."""
|
||||
session, conn, _ = _make_session()
|
||||
_activate(session, conn)
|
||||
|
||||
@@ -2479,8 +2442,7 @@ class TestDispatchErrorHandling:
|
||||
|
||||
def test_internal_error_counter_resets_on_success(self):
|
||||
"""A successful dispatch between errors prevents the threshold
|
||||
from being reached.
|
||||
"""
|
||||
from being reached."""
|
||||
session, conn, _ = _make_session()
|
||||
_activate(session, conn)
|
||||
|
||||
@@ -2523,8 +2485,7 @@ class TestInflightPerReqInvariant:
|
||||
def test_invariant_holds_through_lifecycle(self):
|
||||
"""Run a full submit→complete sequence for two concurrent
|
||||
kv_request_ids and assert the counter matches `_inflight` at
|
||||
every observable step, including the empty-after-finish case.
|
||||
"""
|
||||
every observable step, including the empty-after-finish case."""
|
||||
session, conn, transport = _make_session()
|
||||
_activate(session, conn)
|
||||
|
||||
@@ -2589,8 +2550,7 @@ class TestInflightPerReqInvariant:
|
||||
"""Populate many inflight xfers across many ids; lookup must
|
||||
match the actual presence in `_inflight` for both hits and
|
||||
misses. The whole point of the counter is that this lookup is
|
||||
constant-time, but we assert correctness, not timing.
|
||||
"""
|
||||
constant-time, but we assert correctness, not timing."""
|
||||
session, _, _ = _make_session()
|
||||
for kv_id_idx in range(100):
|
||||
kv_id = f"req-{kv_id_idx}"
|
||||
|
||||
@@ -264,8 +264,7 @@ class TestAsyncLookupManager:
|
||||
subsequent lookup then returns False (MISS) directly, WITHOUT enqueuing
|
||||
a fresh batch_lookup — that is what makes the request unable to loop.
|
||||
The entry is retained (as False) not dropped, so cleanup()'s reverse
|
||||
index stays consistent; an unknown key is a no-op.
|
||||
"""
|
||||
index stays consistent; an unknown key is a no-op."""
|
||||
mgr = InMemoryLookupManager(existing_keys={_key(1), _key(2)})
|
||||
ctx = _ctx("reqA")
|
||||
assert mgr.lookup(_key(1), ctx) is None
|
||||
@@ -305,8 +304,7 @@ class TestAsyncLookupManager:
|
||||
receive at most one result per key. Normal operation resolves a key a
|
||||
single time; a second result for an already-decided key trips the assert
|
||||
that guards the invariant (a silent overwrite could flip a corrected
|
||||
miss back to True and reopen the failed-load livelock).
|
||||
"""
|
||||
miss back to True and reopen the failed-load livelock)."""
|
||||
mgr = InMemoryLookupManager(existing_keys={_key(1)})
|
||||
ctx = _ctx("reqA")
|
||||
|
||||
|
||||
@@ -369,8 +369,7 @@ def test_shutdown_discards_pending_tasks(fs_tier):
|
||||
@pytest.mark.parametrize("use_c_ext", [True, False])
|
||||
def test_store_load_data_integrity(fs_tier, monkeypatch, use_c_ext, batch_size):
|
||||
"""Data written by store must be exactly recovered by load, for batches
|
||||
of any size -- including the empty batch.
|
||||
"""
|
||||
of any size -- including the empty batch."""
|
||||
import vllm.v1.kv_offload.tiering.fs.io as io_mod
|
||||
|
||||
if use_c_ext and not io_mod._HAS_FSIO_C:
|
||||
@@ -467,8 +466,7 @@ def test_wait_idle_blocks_until_tasks_complete():
|
||||
|
||||
def test_batch_lookup_c_extension(tmp_path):
|
||||
"""Validates batch_lookup_C: empty, single, all-existing, all-missing,
|
||||
mixed ordering, and input type validation.
|
||||
"""
|
||||
mixed ordering, and input type validation."""
|
||||
try:
|
||||
from vllm.fs_io_C import batch_lookup as batch_lookup_C
|
||||
except ImportError:
|
||||
@@ -532,8 +530,7 @@ def test_batch_lookup_dispatch(fs_tier, monkeypatch, use_c_ext):
|
||||
@pytest.mark.parametrize("use_c_ext", [True, False])
|
||||
def test_out_of_bounds_block_id_smoke(fs_tier, monkeypatch, use_c_ext):
|
||||
"""Smoke test: a block id beyond the primary tensor's block count must
|
||||
fail the job, for both the C extension and the Python fallback.
|
||||
"""
|
||||
fail the job, for both the C extension and the Python fallback."""
|
||||
import vllm.v1.kv_offload.tiering.fs.io as io_mod
|
||||
|
||||
if use_c_ext and not io_mod._HAS_FSIO_C:
|
||||
@@ -615,8 +612,7 @@ def test_batched_partial_load_failure_keeps_loaded_blocks(
|
||||
loaded before it (#50321). Corrupt the LAST block: the earlier blocks load
|
||||
fine, so the job reports successful_keys for them and marks only the failed
|
||||
tail a miss. The earlier keys stay HIT — including for the same request —
|
||||
while the corrupt block stays a MISS (its file was removed).
|
||||
"""
|
||||
while the corrupt block stays a MISS (its file was removed)."""
|
||||
import vllm.v1.kv_offload.tiering.fs.io as io_mod
|
||||
|
||||
if use_c_ext and not io_mod._HAS_FSIO_C:
|
||||
@@ -668,8 +664,7 @@ def test_batched_load_first_block_fails_marks_whole_batch(
|
||||
):
|
||||
"""When the FIRST block fails, nothing loaded before it: the job reports no
|
||||
successful_keys (None) and the whole batch is marked a miss for the
|
||||
request.
|
||||
"""
|
||||
request."""
|
||||
import vllm.v1.kv_offload.tiering.fs.io as io_mod
|
||||
|
||||
if use_c_ext and not io_mod._HAS_FSIO_C:
|
||||
@@ -700,8 +695,7 @@ def test_transient_load_failure_leaves_file(fs_tier, monkeypatch, use_c_ext):
|
||||
"""A transient host error (here ELOOP on open) is NOT a short read: the job
|
||||
fails but the block file must survive untouched, on both the C and Python
|
||||
paths. Deleting on a transient error would turn a passing hiccup into
|
||||
permanent data loss.
|
||||
"""
|
||||
permanent data loss."""
|
||||
import vllm.v1.kv_offload.tiering.fs.io as io_mod
|
||||
|
||||
if use_c_ext and not io_mod._HAS_FSIO_C:
|
||||
@@ -805,8 +799,7 @@ def test_mixed_job_results_emit_event_only_for_successful_job(
|
||||
fs_tier_with_events, monkeypatch
|
||||
):
|
||||
"""With a failed and a successful store job in flight, exactly one event
|
||||
is emitted and its keys belong to the successful job.
|
||||
"""
|
||||
is emitted and its keys belong to the successful job."""
|
||||
import vllm.v1.kv_offload.tiering.fs.manager as mgr_mod
|
||||
|
||||
tier = fs_tier_with_events
|
||||
@@ -891,8 +884,7 @@ def test_events_require_global_kv_events_flag(tmp_path):
|
||||
|
||||
def test_cascade_store_emits_fs_event_through_tiering_manager(tmp_path):
|
||||
"""A GPU->CPU->fs cascade surfaces the tier-owned FS stored event via the
|
||||
TieringOffloadingManager's aggregated take_events().
|
||||
"""
|
||||
TieringOffloadingManager's aggregated take_events()."""
|
||||
from vllm.v1.kv_offload.tiering.manager import (
|
||||
CPUPrimaryTierOffloadingManager,
|
||||
TieringOffloadingManager,
|
||||
|
||||
@@ -346,8 +346,7 @@ class TestMockObjTierBasic:
|
||||
the scheduler would re-issue the same doomed promotion every step for
|
||||
the life of the request. The mark is served from cache with no
|
||||
re-probe, so even though the mock object is still 'present' the SAME
|
||||
request now resolves to MISS.
|
||||
"""
|
||||
request now resolves to MISS."""
|
||||
ctx = ReqContext(req_id="obj-livelock")
|
||||
self.tier.submit_store(make_job(1, [key(1)], [0]))
|
||||
assert all(r.success for r in drain(self.tier))
|
||||
@@ -651,8 +650,7 @@ class TestObjTierKVEvents:
|
||||
def test_mixed_job_results_emit_event_only_for_successful_job(self):
|
||||
"""With a failed and a successful store job resolving in the same
|
||||
poll, exactly one event is emitted and its keys belong to the
|
||||
successful job.
|
||||
"""
|
||||
successful job."""
|
||||
original = self.agent.check_xfer_state
|
||||
self.agent.check_xfer_state = lambda h: "ERR" if h._id == 0 else original(h)
|
||||
self.tier.submit_store(make_job(1, [key(1)], [0])) # handle 0: fails
|
||||
|
||||
@@ -273,8 +273,7 @@ _CASCADE_SUPPLY_BEHAVIOR = {
|
||||
|
||||
def test_every_lookup_result_has_a_declared_cascade_behavior():
|
||||
"""A new LookupResult must state what the cascade does with it, rather
|
||||
than falling into whichever branch happens to catch it.
|
||||
"""
|
||||
than falling into whichever branch happens to catch it."""
|
||||
assert set(_CASCADE_SUPPLY_BEHAVIOR) == set(LookupResult)
|
||||
|
||||
|
||||
@@ -321,8 +320,7 @@ class TestTieringOffloadingManager:
|
||||
|
||||
def test_failed_promotion_finalizes_primary_with_failure(self, manager_setup):
|
||||
"""A failed promotion still finalizes the primary slots with
|
||||
success=False; the tier corrects its own verdict.
|
||||
"""
|
||||
success=False; the tier corrects its own verdict."""
|
||||
from unittest.mock import patch
|
||||
|
||||
from vllm.v1.kv_offload.tiering.base import JobResult
|
||||
@@ -1093,8 +1091,7 @@ class TestTieringOffloadingManager:
|
||||
|
||||
def test_on_new_request_lifecycle(self, manager_setup):
|
||||
"""Policy defaults to CHUNK_LEVEL, escalates when a tier requests it,
|
||||
and is cleaned up on on_request_finished.
|
||||
"""
|
||||
and is cleaned up on on_request_finished."""
|
||||
# Default: all tiers return CHUNK_LEVEL
|
||||
ctx = ReqContext(req_id="req_policy_lifecycle")
|
||||
result = self.manager.on_new_request(ctx)
|
||||
@@ -1171,8 +1168,7 @@ class TestTieringOffloadingManager:
|
||||
|
||||
def _make_request_level_request(self, req_id: str) -> ReqContext:
|
||||
"""Start a request for which tier1 asks for request-level offloading,
|
||||
with tier1's submit_store wrapped so the cascade can be observed.
|
||||
"""
|
||||
with tier1's submit_store wrapped so the cascade can be observed."""
|
||||
self.secondary_tier1.on_new_request = lambda req_context: (
|
||||
RequestOffloadingContext(policy=OffloadPolicy.REQUEST_LEVEL)
|
||||
)
|
||||
@@ -1194,8 +1190,7 @@ class TestTieringOffloadingManager:
|
||||
def test_cascade_rejects_retry_from_primary(self, manager_setup):
|
||||
"""RETRY would be parked with no guarantee of ever draining, so the
|
||||
cascade refuses it. No real primary tier returns it, so this is the one
|
||||
disposition that has to be forced.
|
||||
"""
|
||||
disposition that has to be forced."""
|
||||
keys = to_keys(range(3))
|
||||
self._start_request()
|
||||
assert self.manager.prepare_store(keys, _CTX) is not None
|
||||
@@ -1210,8 +1205,7 @@ class TestTieringOffloadingManager:
|
||||
|
||||
def _start_in_flight_primary_write(self, keys: list[OffloadKey]) -> ReqContext:
|
||||
"""Leave a GPU->primary write for `keys` open, so they look present to
|
||||
prepare_store but are not yet readable.
|
||||
"""
|
||||
prepare_store but are not yet readable."""
|
||||
writer_ctx = ReqContext(req_id="req_writer")
|
||||
self._start_request(writer_ctx)
|
||||
assert self.manager.prepare_store(keys, writer_ctx) is not None
|
||||
@@ -1220,8 +1214,7 @@ class TestTieringOffloadingManager:
|
||||
def test_cascade_defers_keys_whose_primary_write_is_in_flight(self, manager_setup):
|
||||
"""A key another request is still writing must reach the peer once the
|
||||
write lands, not be dropped: prepare_store already counts it as stored
|
||||
and the scheduler advances past its chunk, so nothing re-offers it.
|
||||
"""
|
||||
and the scheduler advances past its chunk, so nothing re-offers it."""
|
||||
keys = to_keys(range(3))
|
||||
writer_ctx = self._start_in_flight_primary_write(keys)
|
||||
|
||||
@@ -1238,8 +1231,7 @@ class TestTieringOffloadingManager:
|
||||
|
||||
def test_deferred_cascade_holds_request_from_finalization(self, manager_setup):
|
||||
"""A request that finishes with keys still deferred must stay alive, or
|
||||
its tiers are torn down before the peer is ever served.
|
||||
"""
|
||||
its tiers are torn down before the peer is ever served."""
|
||||
keys = to_keys(range(3))
|
||||
writer_ctx = self._start_in_flight_primary_write(keys)
|
||||
|
||||
@@ -1262,8 +1254,7 @@ class TestTieringOffloadingManager:
|
||||
self, manager_setup
|
||||
):
|
||||
"""A failed write frees the chunk, so the deferred key resolves to MISS
|
||||
and the request finalizes instead of parking forever.
|
||||
"""
|
||||
and the request finalizes instead of parking forever."""
|
||||
keys = to_keys(range(3))
|
||||
writer_ctx = self._start_in_flight_primary_write(keys)
|
||||
|
||||
@@ -1283,8 +1274,7 @@ class TestTieringOffloadingManager:
|
||||
def test_reset_cache_clears_orchestrator_state(self, manager_setup):
|
||||
"""reset_cache wipes every kind of orchestrator state and resets
|
||||
primary tier; pending submissions are dropped without being sent
|
||||
to the secondary tier. Active request state is retained.
|
||||
"""
|
||||
to the secondary tier. Active request state is retained."""
|
||||
# Cascade — populates primary chunks and leaves cascade jobs
|
||||
# in _jobs (the synchronous example tier has already
|
||||
# queued completions); reset_cache's drain loop will pick them up.
|
||||
@@ -1376,8 +1366,7 @@ class TestTieringOffloadingManager:
|
||||
self, manager_setup, load_tier_filter
|
||||
):
|
||||
"""Filter excluding secondary medium returns MISS from secondaries
|
||||
even when they hold the chunk; primary is unaffected.
|
||||
"""
|
||||
even when they hold the chunk; primary is unaffected."""
|
||||
chunks = to_keys(range(2))
|
||||
# Put one chunk in primary, one only in secondary
|
||||
self._start_request()
|
||||
|
||||
@@ -1344,8 +1344,7 @@ def test_thinking_budget_invalid_budget_rejected(invalid_budget):
|
||||
|
||||
def test_thinking_budget_long_thinking_section_end_marker_found_at_correct_index():
|
||||
"""Test thinking budget enforced for a long thinking run,
|
||||
then a natural end marker.
|
||||
"""
|
||||
then a natural end marker."""
|
||||
h = ThinkingBudgetStateHolder(
|
||||
MockReasoningConfig(), 8, 0, torch.device("cpu"), False
|
||||
)
|
||||
|
||||
@@ -138,8 +138,7 @@ class EntryPoints(list):
|
||||
|
||||
class DummyPerReqLogitsProcessor:
|
||||
"""The request-level logits processor masks out all logits except the
|
||||
token id identified by `target_token`
|
||||
"""
|
||||
token id identified by `target_token`"""
|
||||
|
||||
def __init__(self, target_token: int) -> None:
|
||||
"""Specify `target_token`."""
|
||||
@@ -158,8 +157,7 @@ class DummyPerReqLogitsProcessor:
|
||||
|
||||
class WrappedPerReqLogitsProcessor(AdapterLogitsProcessor):
|
||||
"""Example of wrapping a fake request-level logit processor to create a
|
||||
batch-level logits processor
|
||||
"""
|
||||
batch-level logits processor"""
|
||||
|
||||
def is_argmax_invariant(self) -> bool:
|
||||
return False
|
||||
|
||||
@@ -1241,8 +1241,7 @@ def test_mla_per_gpu_with_pipeline_parallelism():
|
||||
|
||||
def test_mla_model_metrics_excludes_standard_attention():
|
||||
"""Test that ModelMetrics uses MLAAttentionMetrics, not AttentionMetrics,
|
||||
for DeepSeek MLA models.
|
||||
"""
|
||||
for DeepSeek MLA models."""
|
||||
hf_config = DeepseekV3Config(
|
||||
hidden_size=7168,
|
||||
num_attention_heads=128,
|
||||
|
||||
@@ -45,8 +45,7 @@ def test_engine_log_metrics_ray(
|
||||
max_tokens: int,
|
||||
) -> None:
|
||||
"""Simple smoke test, verifying this can be used without exceptions.
|
||||
Need to start a Ray cluster in order to verify outputs.
|
||||
"""
|
||||
Need to start a Ray cluster in order to verify outputs."""
|
||||
|
||||
@ray.remote(num_gpus=1)
|
||||
class EngineTestActor:
|
||||
@@ -120,8 +119,7 @@ def test_sanitized_opentelemetry_name():
|
||||
def _install_mock_metric(wrapper: RayPrometheusMetric) -> MagicMock:
|
||||
"""Swap the wrapper's underlying Ray metric for a MagicMock while
|
||||
preserving the real metric's ``_tag_keys`` (labels() reads them to
|
||||
validate arity).
|
||||
"""
|
||||
validate arity)."""
|
||||
real_metric = wrapper.metric
|
||||
mock = MagicMock()
|
||||
mock._tag_keys = real_metric._tag_keys
|
||||
@@ -131,8 +129,7 @@ def _install_mock_metric(wrapper: RayPrometheusMetric) -> MagicMock:
|
||||
|
||||
def test_ray_counter_labels_returns_independent_children():
|
||||
"""RayCounterWrapper.labels() must return distinct labeled children that
|
||||
each carry their own tag set.
|
||||
"""
|
||||
each carry their own tag set."""
|
||||
base = RayCounterWrapper(
|
||||
name="vllm_test_finish_reason",
|
||||
documentation="",
|
||||
@@ -152,8 +149,7 @@ def test_ray_counter_labels_returns_independent_children():
|
||||
|
||||
def test_ray_counter_inc_forwards_per_child_tags():
|
||||
""".inc() on a labeled counter must forward that child's tags to the
|
||||
underlying Ray metric (not rely on a shared set_default_tags).
|
||||
"""
|
||||
underlying Ray metric (not rely on a shared set_default_tags)."""
|
||||
wrapper = RayCounterWrapper(
|
||||
name="vllm_test_counter_tag_forward",
|
||||
documentation="",
|
||||
@@ -218,8 +214,7 @@ def test_ray_histogram_labels_returns_independent_children_and_forwards_tags():
|
||||
def test_ray_counter_labels_accepts_non_string_label_values():
|
||||
"""RayPrometheusStatLogger passes ``str(idx)`` for engine indexes; this
|
||||
covers the coercion path for any caller that passes a non-string label
|
||||
value positionally.
|
||||
"""
|
||||
value positionally."""
|
||||
wrapper = RayCounterWrapper(
|
||||
name="vllm_test_nonstr_label",
|
||||
documentation="",
|
||||
@@ -242,8 +237,7 @@ def test_ray_counter_labels_arity_validation():
|
||||
|
||||
def test_unlabeled_inc_carries_replica_id():
|
||||
"""Recording on an unlabeled metric must still pass ReplicaId — it's a
|
||||
declared tag_key and Ray rejects updates that omit any declared key.
|
||||
"""
|
||||
declared tag_key and Ray rejects updates that omit any declared key."""
|
||||
wrapper = RayCounterWrapper(
|
||||
name="vllm_test_unlabeled_replica_id",
|
||||
documentation="",
|
||||
@@ -256,8 +250,7 @@ def test_unlabeled_inc_carries_replica_id():
|
||||
|
||||
def test_double_labels_raises():
|
||||
"""labels() on an already-labeled child should raise, mirroring the
|
||||
prometheus_client contract.
|
||||
"""
|
||||
prometheus_client contract."""
|
||||
wrapper = RayCounterWrapper(
|
||||
name="vllm_test_double_labels",
|
||||
documentation="",
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Test that batched_count_greater_than does not trigger 0/1 specialization
|
||||
recompiles when batch_size varies.
|
||||
"""
|
||||
recompiles when batch_size varies."""
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user