Add pydocstyle to the ruff rules (#52136)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Signed-off-by: Harry Mellor <hej.mellor@gmail.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-09-16 16:44:29 +00:00
committed by GitHub
co-authored by Claude Opus 5
parent 804e5377aa
commit 8c1557a79c
2002 changed files with 15024 additions and 14269 deletions
@@ -2,7 +2,8 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Fine-grained partial prefix-cache hits for hybrid (full attention + mamba
"align") models: scheduler chunk splitting, partial tail registration, CoW
on partial hits, and same-step deferral."""
on partial hits, and same-step deferral.
"""
from math import lcm
from types import SimpleNamespace
@@ -174,7 +175,8 @@ def test_mamba_align_split_partial_tail_schedule(dcp_world_size: int):
"""Chunk ends with partial hits on: block-aligned chunks, one extra stop
at the prompt's last hash boundary (registering the partial tail), then
the remaining tokens. block=512, hash=32, prompt=10000, budget=8192:
0 -> 8192 -> 9728 -> 9984 -> 10000."""
0 -> 8192 -> 9728 -> 9984 -> 10000.
"""
block_size = 512
scheduler_block_size = block_size * dcp_world_size
hash_block_size = 32
@@ -742,7 +744,8 @@ def test_hash_aligned_query_end_uses_regular_partial_tail():
def test_external_mamba_hit_same_block_uses_running_cow_on_continue():
"""An external mid-block hit must become a running request even when its
first continuation does not need another Mamba block."""
first continuation does not need another Mamba block.
"""
hash_block_size = 2
mamba_block_size = 4 * hash_block_size
kv_cache_config = KVCacheConfig(
@@ -814,7 +817,8 @@ def test_external_mamba_hit_same_block_uses_running_cow_on_continue():
def test_boundary_state_offloads_returns_cow_target():
"""Boundary hand-offs expose aligned snapshots and the partial-tail CoW
target, never the overwritten CoW source."""
target, never the overwritten CoW source.
"""
hash_block_size = 2
block_size = 2 * hash_block_size
kv_cache_config = KVCacheConfig(
@@ -1000,7 +1004,8 @@ def test_connector_finish_registers_partial_tail_before_cleanup():
def test_boundary_state_offload_dropped_when_request_freed_before_drain():
"""A hand-off recorded in the same scheduling pass as the request's death
must not be drained: its release hook has already run, so draining would
leak a pinned block."""
leak a pinned block.
"""
hash_block_size = 2
block_size = 2 * hash_block_size
kv_cache_config = KVCacheConfig(
@@ -1047,7 +1052,8 @@ def test_boundary_state_offload_dropped_when_request_freed_before_drain():
def test_boundary_state_offloads_block_aligned_prompt():
"""A prompt ending on a block boundary registers no CoW partial tail; its
boundary state block is handed off as a snapshot instead (once)."""
boundary state block is handed off as a snapshot instead (once).
"""
hash_block_size = 2
block_size = 2 * hash_block_size
kv_cache_config = KVCacheConfig(
@@ -1101,7 +1107,8 @@ def test_boundary_state_offloads_block_aligned_prompt():
def test_truncate_computed_blocks_preserves_sparse_prefix_positions():
"""truncate_computed_blocks slices each group by its own block size,
keeps null placeholders in the retained prefix, and leaves the original
lookup result untouched (pure view, no refcount changes)."""
lookup result untouched (pure view, no refcount changes).
"""
hash_block_size = 2
kv_cache_config = KVCacheConfig(
num_blocks=24,
@@ -1156,7 +1163,8 @@ def test_truncate_computed_blocks_preserves_sparse_prefix_positions():
def test_truncate_computed_blocks_allows_short_mamba_group_only():
"""External state may replace a short Mamba hit, but other groups must
cover the aligned local endpoint."""
cover the aligned local endpoint.
"""
hash_block_size = 2
kv_cache_config = KVCacheConfig(
num_blocks=24,
@@ -1298,7 +1306,8 @@ def test_hybrid_mamba_partial_tail_owner_continue_preserves_later_hit():
def test_hybrid_mamba_moved_partial_entry_defers_same_step_hit():
"""The owner's move re-arms the same-step guard: the moved entry is
filled by this step's copy, and chained same-step copies read stale
sources, so a request hitting it in the move step must be deferred."""
sources, so a request hitting it in the move step must be deferred.
"""
hash_block_size = 2
block_size = 2 * hash_block_size
kv_cache_config = KVCacheConfig(
@@ -1588,7 +1597,8 @@ def test_hybrid_partial_hash_truncates_full_attention_hit_length():
def test_cow_retained_blocks_returned_for_release():
"""new_step_starts returns the CoW copy retentions instead of freeing
them; the scheduler owns releasing them once the copy has run."""
them; the scheduler owns releasing them once the copy has run.
"""
hash_block_size = 2
block_size = 2 * hash_block_size
kv_cache_config = KVCacheConfig(
@@ -1641,7 +1651,8 @@ def test_cow_retained_blocks_returned_for_release():
def test_free_cow_retained_blocks_defers_until_copy_step_processed():
"""Scheduler releases CoW retentions immediately when the copy's step has
been processed (or deferral is off), and defers them otherwise."""
been processed (or deferral is off), and defers them otherwise.
"""
from collections import deque
freed: list = []
@@ -1678,7 +1689,8 @@ def test_free_cow_retained_blocks_defers_until_copy_step_processed():
def test_full_attention_eagle_drops_one_hash_unit():
"""With fine-grained partial hits, eagle rewinds the hit by one hash unit
instead of a whole cache block: the tail block's KV is append-only, so it
still covers the reduced length and stays in the hit as a partial block."""
still covers the reduced length and stays in the hit as a partial block.
"""
from vllm.v1.core.block_pool import BlockPool
from vllm.v1.core.single_type_kv_cache_manager import FullAttentionManager
@@ -1754,7 +1766,8 @@ def test_hybrid_partial_hit_with_eagle_stays_within_group_blocks():
"""Regression: with eagle, the mamba group must not receive the eagle
lookup margin — its finder never applies the drop, so it could return a
hit past the blocks the (dropped) full-attention group covers, crashing
the consumer's CoW with block_idx >= len(req_blocks)."""
the consumer's CoW with block_idx >= len(req_blocks).
"""
hash_block_size = 2
block_size = 2 * hash_block_size
kv_cache_config = KVCacheConfig(
@@ -2356,7 +2369,8 @@ def test_retention_snapshots_handed_off_with_exact_block_ids():
"""Under sparse retention, each retained mamba boundary state block is
handed to the connector with its exact block id. Connectors must not resolve
align-mode state blocks positionally because the block table is not
append-only."""
append-only.
"""
hash_block_size = 2
block_size = 4
manager = make_kv_cache_manager(
@@ -2397,7 +2411,8 @@ def test_snapshot_handoff_dense_default_retention():
"""With dense (default) retention, every materialized mamba boundary
state block is handed off — regular mamba-align + prefix-match-unit
deployments get store-able boundary snapshots without setting a
prefix-cache retention interval."""
prefix-cache retention interval.
"""
hash_block_size = 2
block_size = 4
manager = make_kv_cache_manager(
@@ -2429,7 +2444,8 @@ def test_snapshot_handoff_dense_default_retention():
def _run_chunked_prefill(manager, req, chunk_size, num_chunks):
"""Prefill ``req`` one ``chunk_size`` chunk per scheduler step, yielding the
boundary-state hand-offs offered (and claimed) at each step."""
boundary-state hand-offs offered (and claimed) at each step.
"""
computed_blocks, num_computed, _ = manager.get_computed_blocks(req)
for step in range(num_chunks):
manager.new_step_starts()
@@ -2493,7 +2509,8 @@ def test_boundary_states_offered_past_prompt_for_resumed_prefill():
``num_prompt_tokens``. A resumed request re-prefills its generated tokens
and every group re-saves them, so filtering on the original prompt length
would silently strip the mamba key for boundaries full attention still
stores; only the connector knows where its save window ends."""
stores; only the connector knows where its save window ends.
"""
block_size = 8
prompt_len = block_size
manager = make_kv_cache_manager(
+4 -2
View File
@@ -552,7 +552,8 @@ def _create_async_pp_scheduler(
def _assert_ordered_subset(delivered: list[int], emitted: list[int]) -> None:
"""Delivered tokens must be an order-preserving subset of the emitted
tokens with no duplicates (tokens are globally unique)."""
tokens with no duplicates (tokens are globally unique).
"""
it = iter(emitted)
for token in delivered:
assert token in it, f"token {token} delivered out of order or twice"
@@ -562,7 +563,8 @@ def _assert_positions_consistent(req, engine: PipelinedEngine) -> None:
"""The i-th delivered output token must be one the runner sampled for
exactly sequence position prompt_len + i: catches a preempted request's
stale output landing on a position the resumed request resampled (or
vice versa), which token-stream equality alone cannot see."""
vice versa), which token-stream equality alone cannot see.
"""
for i, token in enumerate(req.output_token_ids):
expected = req.num_prompt_tokens + i
actual = engine.emitted_position[token]
+12 -6
View File
@@ -178,7 +178,8 @@ def _shared_layout_config():
class TestCSALinearGrouping:
"""A CSA + linear-attention model (sparse attention with a compressor ring,
plus sharded GDN and one TP-replicated PLE state) goes through the generic
packed-group path; no model-specific branch is involved."""
packed-group path; no model-specific branch is involved.
"""
@staticmethod
def _mamba_groups(groups):
@@ -193,7 +194,8 @@ class TestCSALinearGrouping:
def test_replicated_state_gets_its_own_group(self):
"""The PLE state is TP-replicated and a different size, so it must not
share a manager group with the sharded GDN states (the NIXL worker
requires exactly one single-layer replicated group)."""
requires exactly one single-layer replicated group).
"""
groups = get_kv_cache_groups(_shared_layout_config(), _make_csa_linear_specs())
mamba_groups = self._mamba_groups(groups)
@@ -246,7 +248,8 @@ class TestCSALinearGrouping:
def test_scratch_group_survives_computed_block_truncation(self):
"""The scratch group contributes no computed blocks, so truncating a
lookup result must skip it: its block size is the ring capacity, which
neither divides the hit length nor bounds the (empty) block list."""
neither divides the hit length nor bounds the (empty) block list.
"""
config = _shared_layout_config()
config.cache_config.enable_prefix_caching = True
groups = get_kv_cache_groups(config, _make_csa_linear_specs())
@@ -360,7 +363,8 @@ class TestCSALinearGrouping:
per page size, which is emitted whole, or a single-layer state that
cannot be split at all -- the mamba layers must be sized against it.
Sizing them against a narrower bucket splits them past what the block
already fits, spending a pool block per extra group for no saving."""
already fits, spending a pool block per extra group for no saving.
"""
config = _mock_vllm_config("BLNHC")
config.speculative_config = None
specs = {}
@@ -405,7 +409,8 @@ class TestSlidingWindowBucketCap:
unbalanced 8-layer paged MLA bucket. Left whole, the SWA bucket would
set the block width and the MLA group would fill under a third of
every block; capped, it splits into groups no wider than the MLA
page."""
page.
"""
config = _shared_layout_config()
specs: dict[str, KVCacheSpec] = {}
for layer, ratio in ((2, 2), (8, 2), (14, 2), (20, 1)):
@@ -606,7 +611,8 @@ class TestCompressorRingGroup:
"""DeepSeek-V4.1 compressor state: a CircularBufferSpec ring (capacity
8) beside 128-token paged MLA groups and 64-token SWA groups. The ring
claims one block per request, never hashes, and does not disturb the
scheduler/hash block sizes the paged groups imply."""
scheduler/hash block sizes the paged groups imply.
"""
config = _shared_layout_config()
config.cache_config.enable_prefix_caching = True
specs: dict[str, KVCacheSpec] = {
+6 -3
View File
@@ -367,7 +367,8 @@ def test_free_request_with_duplicate_mm_hashes():
"""Freeing a request whose two inputs share the same mm_hash must fully
clean up request_cached_ids. After the first free_encoder_input call,
cached[mm_hash] becomes empty; the second call must still remove the
remaining input_id from request_cached_ids."""
remaining input_id from request_cached_ids.
"""
manager = EncoderCacheManager(cache_size=20)
req = MockRequest("r1", ["imgA", "imgA"], [4, 4])
@@ -383,7 +384,8 @@ def test_free_request_with_duplicate_mm_hashes():
def test_duplicate_mm_hash_stays_referenced_until_last_free():
"""`cached` holds one reference per request, not per position, so freeing
the first of two occurrences must not release the entry."""
the first of two occurrences must not release the entry.
"""
manager = EncoderCacheManager(cache_size=20)
req = MockRequest("r1", ["imgA", "imgA"], [4, 4])
@@ -404,7 +406,8 @@ def test_duplicate_mm_hash_stays_referenced_until_last_free():
def test_duplicate_mm_hash_is_not_evicted_before_last_use():
"""An item repeated within one request (an image carried across
conversation turns) must not become reclaimable capacity while a later
occurrence still has to be spliced in, or the encoder recomputes it."""
occurrence still has to be spliced in, or the encoder recomputes it.
"""
manager = EncoderCacheManager(cache_size=8)
req = MockRequest("r1", ["imgA", "imgA"], [4, 4])
other = MockRequest("r2", ["imgB"], [8])
+24 -19
View File
@@ -1069,9 +1069,7 @@ def _stats(requests: int, queries: int, hits: int) -> PrefixCacheStats:
def test_metrics():
"""
Test the prefix caching metrics.
"""
"""Test the prefix caching metrics."""
metrics = CachingMetrics(max_recent_requests=5)
assert metrics.hit_rate == 0.0
@@ -1101,9 +1099,7 @@ def test_metrics():
def test_metrics_empty_stats():
"""
Test the prefix caching metrics with empty stats.
"""
"""Test the prefix caching metrics with empty stats."""
metrics = CachingMetrics(max_recent_requests=5)
metrics.observe(_stats(0, 0, 0))
metrics.observe(_stats(1, 20, 9))
@@ -1903,7 +1899,7 @@ def test_get_max_concurrency_for_kv_cache_config():
def test_allocate_with_lookahead():
"""Verify that lookahead tokens correctly affect block allocation"""
"""Verify that lookahead tokens correctly affect block allocation."""
block_size = 4
config = KVCacheConfig(
num_blocks=10,
@@ -2597,7 +2593,8 @@ def test_get_kv_cache_config_kpool_tail_coowns_indexer_tensor():
"""The kpool tail parasitizes the indexer tensors instead of getting its
own: sibling idx/tail tensors paired by layer order, zero standalone tail
bytes, one shared block per request, and no prefix-caching leakage from
the kpool-sized scratch group."""
the kpool-sized scratch group.
"""
model_config = ModelConfig(max_model_len=8192)
vllm_config = VllmConfig(model_config=model_config)
@@ -2668,7 +2665,8 @@ def test_get_kv_cache_config_kpool_tail_coowns_indexer_tensor():
def test_glm5_kpool_tail_does_not_drag_hash_block_size():
"""The tail's kpool-sized scratch block (4 tokens) must not constrain the
prefix-cache hash granularity: participating groups alone decide it."""
prefix-cache hash granularity: participating groups alone decide it.
"""
model_config = ModelConfig(max_model_len=8192)
vllm_config = VllmConfig(model_config=model_config)
@@ -2744,7 +2742,8 @@ def test_get_kv_cache_config_mamba_hybrid_sharing_infeasible_no_indexer():
def test_get_kv_cache_config_mamba_hybrid_sharing_prepadded_mamba():
"""Platform-prepadded mamba pages (mamba_page_size_padded hint) must not
disable slot sharing; the layout re-pads them to the MLA page."""
disable slot sharing; the layout re-pads them to the MLA page.
"""
model_config = ModelConfig(max_model_len=8192)
vllm_config = VllmConfig(model_config=model_config)
@@ -2775,7 +2774,8 @@ def test_get_kv_cache_config_mamba_hybrid_sharing_prepadded_mamba():
def test_get_kv_cache_config_mamba_hybrid_sharing_pp_balanced_projection():
"""Round-robin mamba grouping keeps practical PP splits balanced: every
stage's largest projected mamba group slice fits its projected MLA
layers, so per-stage slot tensors all have an MLA owner."""
layers, so per-stage slot tensors all have an MLA owner.
"""
model_config = ModelConfig(max_model_len=8192)
vllm_config = VllmConfig(model_config=model_config)
@@ -2819,7 +2819,8 @@ def test_get_kv_cache_config_mamba_hybrid_sharing_pp_group_count_bump(monkeypatc
"""PP=4's default partition [11,11,12,11] leaves stage 0 with 2 MLA
layers but a round-robin slice of 3 under the minimum 4 mamba groups;
grouping bumps to 5 groups so every stage's projection keeps sharing
on instead of silently falling back on stage 0."""
on instead of silently falling back on stage 0.
"""
from vllm.config import ParallelConfig
from vllm.distributed.utils import get_pp_indices
@@ -2868,7 +2869,8 @@ def test_get_kv_cache_config_mamba_hybrid_sharing_pp_starved_stage(monkeypatch):
def test_get_kv_cache_config_mamba_hybrid_sharing_beats_cross_layers_flag():
"""Hybrid slot sharing must take precedence over the experimental
enable_cross_layers_blocks packed layout: generic packing would give the
MLA slots a strided view, breaking the contiguous virtual split."""
MLA slots a strided view, breaking the contiguous virtual split.
"""
model_config = ModelConfig(max_model_len=8192)
kv_transfer_config = KVTransferConfig(
kv_connector="NixlConnector",
@@ -3170,7 +3172,8 @@ def test_multi_run_layer_compact_strides_place_hoisted_heads():
"""A layer-compact run region is its own dense allocation: under LHBNC the head
groups sit between the layers and the blocks, so a run's block stride is one head
group's slice, not the whole page (regression test for setStorage out-of-bounds
on ROCm hybrid models)."""
on ROCm hybrid models).
"""
full = new_kv_cache_spec()
swa = new_sliding_window_spec(sliding_window=full.block_size * 2)
page = full.page_size_bytes
@@ -3249,7 +3252,8 @@ def test_mla_with_incompatible_swa_uses_one_full_allocation_group(caplog_vllm):
def test_hidden_states_with_tp_scales_page_size():
"""When TP shrinks KV pages below the hidden-state per-token cost,
get_kv_cache_groups must scale up target block sizes so that the
common page accommodates the unsharded hidden states."""
common page accommodates the unsharded hidden states.
"""
# Simulate TP=4 sharding a model with 8 KV heads → 2 per rank.
# KV page = block_size(16) * num_kv_heads(2) * head_size(64) * dtype(2)
# = 16 * 2 * 64 * 2 = 4096 bytes.
@@ -3922,8 +3926,7 @@ def test_unify_kv_cache_spec_page_size_mamba():
def test_hma_not_disabled_when_kv_events_enabled():
"""
Test enabling KV events must not force disable_hybrid_kv_cache_manager to True.
"""Test enabling KV events must not force disable_hybrid_kv_cache_manager to True.
This test guards against that regression by verifying that a VllmConfig
with kv_events_config set still resolves disable_hybrid_kv_cache_manager
@@ -4080,7 +4083,8 @@ def test_check_enough_kv_cache_memory_reserves_null_block():
def test_is_full_attention_spec_unwraps_uniform_type_specs():
"""``UniformTypeKVCacheSpecs`` is not a ``FullAttentionSpec``, so callers
scanning groups with a bare isinstance miss DeepSeek-V4-shaped configs."""
scanning groups with a bare isinstance miss DeepSeek-V4-shaped configs.
"""
common = dict(num_kv_heads=1, head_size=1, dtype=torch.float32)
full = FullAttentionSpec(block_size=4, **common)
mla = MLAAttentionSpec(block_size=4, **common)
@@ -4249,7 +4253,8 @@ def test_no_warning_when_draft_group_is_identified(caplog_vllm):
def _deepseek_v4_specs(model_version="deepseek_v4"):
"""DeepseekV4-shaped specs: full MLA layers plus sliding-window MLA layers
at differing page sizes, with the MTP draft layer registered last."""
at differing page sizes, with the MTP draft layer registered last.
"""
return {
"model.layers.0.self_attn.attn": new_mla_spec(),
"model.layers.1.self_attn.attn": new_mla_spec(),
+3 -6
View File
@@ -17,8 +17,7 @@ def new_kv_cache_spec():
def test_initialize_kv_cache_for_kv_sharing_different_attn_groups():
"""
Test initializing KV cache sharing with different attention groups.
"""Test initializing KV cache sharing with different attention groups.
Layers in the same KV cache group might be placed in different attn groups
if they have different attention backends.
"""
@@ -50,8 +49,7 @@ def test_initialize_kv_cache_for_kv_sharing_different_attn_groups():
def test_initialize_kv_cache_for_kv_sharing_same_attn_groups():
"""
Test case assuming that all layers in the same KV cache group have the same
"""Test case assuming that all layers in the same KV cache group have the same
attention backends. This is true for most models.
"""
shared_kv_cache_layers = {
@@ -79,8 +77,7 @@ def test_initialize_kv_cache_for_kv_sharing_same_attn_groups():
def test_initialize_kv_cache_for_kv_sharing_no_attn_groups():
"""
Test KV sharing set up when no attention groups are provided.
"""Test KV sharing set up when no attention groups are provided.
This is the case for the TPU model runner, which doesn't have
support for attention groups yet.
"""
+6 -3
View File
@@ -55,7 +55,8 @@ def _mm_feature(offset: int, length: int) -> MultiModalFeatureSpec:
def test_strip_covered_mm_data() -> None:
"""Items fully inside the computed prefix lose their data; items touching
the uncomputed region keep it; already-None data stays None."""
the uncomputed region keep it; already-None data stays None.
"""
from dataclasses import replace
covered = _mm_feature(offset=0, length=100)
@@ -107,7 +108,8 @@ def _mm_feature_mixed(offset: int, length: int) -> MultiModalFeatureSpec:
def test_strip_covered_mm_data_mrope() -> None:
"""For M-RoPE models, covered items keep their keep_on_cpu metadata fields
(the worker needs them to compute positions); payload fields are dropped."""
(the worker needs them to compute positions); payload fields are dropped.
"""
covered = _mm_feature_mixed(offset=0, length=100)
uncovered = _mm_feature_mixed(offset=300, length=100)
@@ -127,7 +129,8 @@ def test_strip_covered_mm_data_shm_address_item() -> None:
"""SHM address descriptors must survive stripping even when covered: the
worker needs the address to resolve the payload and to acknowledge the
sender's writer reference. Stripping it crashed the SHM receiver cache on
the second identical request (vllm-project/vllm#54994)."""
the second identical request (vllm-project/vllm#54994).
"""
address_item = MultiModalKwargsItem(
{
"address": MultiModalFieldElem(data=4096, field=MultiModalBatchedField()),
+64 -56
View File
@@ -126,7 +126,8 @@ def make_kv_cache_manager(kv_cache_config: KVCacheConfig, **kwargs) -> KVCacheMa
"""Build a ``KVCacheManager``, deriving ``scheduler_block_size`` from the
config (LCM of group block sizes) unless explicitly provided. This mirrors
``resolve_kv_cache_block_sizes`` for the non-context-parallel case used by
these tests, so callers don't have to pass it at every site."""
these tests, so callers don't have to pass it at every site.
"""
kwargs.setdefault(
"scheduler_block_size",
lcm(*(g.kv_cache_spec.block_size for g in kv_cache_config.kv_cache_groups)),
@@ -1826,8 +1827,7 @@ def _test_partial_request_hit(
def _make_hybrid_kv_cache_config(
block_size: int, num_blocks: int, spec_types: list[str]
) -> KVCacheConfig:
"""
Create a KVCacheConfig with the specified spec types.
"""Create a KVCacheConfig with the specified spec types.
Args:
block_size: The block size for KV cache.
@@ -1837,6 +1837,7 @@ def _make_hybrid_kv_cache_config(
- "sliding_window": SlidingWindowSpec with window=2*block_size
- "sliding_window_large": SlidingWindowSpec with window=4*block_size
- "mamba": MambaSpec
"""
spec_map = {
"full": lambda: FullAttentionSpec(
@@ -1938,8 +1939,7 @@ _HYBRID_MODEL_TEST_CASES = [
@pytest.mark.parametrize("spec_types", _HYBRID_MODEL_TEST_CASES)
def test_prefill_hybrid_model_combinations(spec_types: list[str]):
"""
Test prefix caching with hybrid models containing various combinations of
"""Test prefix caching with hybrid models containing various combinations of
KV cache spec types.
This unified test covers:
@@ -2021,8 +2021,7 @@ _EAGLE_HYBRID_MODEL_TEST_CASES = [
def test_prefill_hybrid_model_combinations_eagle(
spec_types: list[str], expect_hit_length: int
):
"""
Test prefix caching with hybrid models (1 full attn + 1 other) with EAGLE.
"""Test prefix caching with hybrid models (1 full attn + 1 other) with EAGLE.
More complex hybrid models with EAGLE are not yet supported (see issue #32802).
"""
block_size = 16
@@ -2489,8 +2488,7 @@ def test_evict():
def test_hash_block_correct_reuse():
"""
This tests when a previously cached block is reused as a new block,
"""This tests when a previously cached block is reused as a new block,
its hash metadata should be correctly reset.
"""
block_size = 16
@@ -2530,8 +2528,7 @@ def test_hash_block_correct_reuse():
def test_computed_blocks_not_evicted():
"""
Test that the computed blocks are not evicted when getting new blocks
"""Test that the computed blocks are not evicted when getting new blocks
for a request if there are any other free blocks.
"""
block_size = 16
@@ -2590,9 +2587,7 @@ def test_computed_blocks_not_evicted():
def test_basic_prefix_caching_disabled():
"""
This tests that the prefix caching is disabled.
"""
"""This tests that the prefix caching is disabled."""
block_size = 4
manager = make_kv_cache_manager(
make_kv_cache_config(block_size, 5),
@@ -2639,11 +2634,9 @@ def test_basic_prefix_caching_disabled():
@pytest.mark.parametrize("hash_fn", [sha256, sha256_cbor])
def test_cache_blocks(hash_fn):
"""
This is a unit test that tests the correctness of the _cache_full_blocks
"""This is a unit test that tests the correctness of the _cache_full_blocks
function of KVCacheManager.
"""
block_size = 4
block_pool = BlockPool(
num_gpu_blocks=5,
@@ -2688,9 +2681,7 @@ def test_cache_blocks(hash_fn):
def test_cache_blocks_multi_group():
"""
This tests that blocks are cached correctly for different kv cache groups.
"""
"""This tests that blocks are cached correctly for different kv cache groups."""
block_size = 4
block_pool = BlockPool(
num_gpu_blocks=10, enable_caching=True, hash_block_size=block_size
@@ -2773,10 +2764,7 @@ def test_cache_blocks_multi_group():
def test_mm_prefix_caching():
"""
This tests that the multi-modal prefix caching is correct.
"""
"""This tests that the multi-modal prefix caching is correct."""
block_size = 16
manager = make_kv_cache_manager(
make_kv_cache_config(block_size, 11),
@@ -2881,8 +2869,7 @@ def test_mm_prefix_caching():
def test_cache_key_salting():
"""
This tests that cache salts are applied during hashing and the cache
"""This tests that cache salts are applied during hashing and the cache
is separated cache as expected.
"""
block_size = 16
@@ -2961,8 +2948,7 @@ def test_cache_key_salting():
def test_prefill_not_enough_free_blocks_with_computed_blocks():
"""
This is a unit test that tests the correctness of the allocate_slots
"""This is a unit test that tests the correctness of the allocate_slots
when there is not enough free blocks. Specifically, when a request
has computed blocks but cannot be allocated due to not enough free blocks,
the computed blocks should not be touched.
@@ -3344,7 +3330,8 @@ def test_kv_cache_events_with_lora(blocks_to_cache: int):
@pytest.mark.parametrize("group_id", [0, 1, 2])
def test_block_stored_event_group_idx(group_id: int):
"""Test BlockStored events emitted by cache_full_blocks carry the correct
group_idx."""
group_idx.
"""
block_size = 4
num_tokens = block_size * 2
@@ -3462,8 +3449,7 @@ def test_session_id_does_not_affect_prefix_cache_identity():
def test_block_stored_event_group_idx_multiple_groups():
"""
Test BlockStored events for separate HMA groups that each carry the
"""Test BlockStored events for separate HMA groups that each carry the
correct group_idx.
Simulates the HMA scenario where full-attention blocks (group 0) and
@@ -3585,8 +3571,7 @@ def test_block_stored_event_group_idx_out_of_bounds(monkeypatch):
@pytest.mark.parametrize("group_id", [0, 1, 2])
def test_block_removed_event_group_idx(group_id: int):
"""
Test BlockRemoved events emitted on eviction carry the group_idx extracted
"""Test BlockRemoved events emitted on eviction carry the group_idx extracted
from the evicted block's BlockHashWithGroupId via get_group_id().
"""
block_size = 4
@@ -3639,7 +3624,8 @@ def test_block_removed_event_group_idx(group_id: int):
def test_emit_cached_block_events():
"""emit_cached_block_events emits one BlockStored for already-cached
(reused) prefix blocks, carrying the correct group_idx /
parent_block_hash / token_ids, and without mutating block state."""
parent_block_hash / token_ids, and without mutating block state.
"""
block_size = 4
num_cached_blocks = 3
kv_cache_group_id = 1
@@ -3752,7 +3738,8 @@ def test_emit_cached_block_events_zero_cached():
def test_eagle_enabled_removes_last_block():
"""Verify Eagle does NOT remove blocks when request
length is divisible by block size."""
length is divisible by block size.
"""
block_size = 16
manager = make_kv_cache_manager(
make_kv_cache_config(block_size, num_blocks=10),
@@ -4155,7 +4142,8 @@ def test_hybrid_cache_blocks_swa_tail_window_only():
(its right-to-left scan stops once a contiguous match is found). Blocks
earlier in the segment can never serve a hit, so
``HybridKVCacheCoordinator.cache_blocks`` should skip them rather than
polluting the prefix-cache hash map."""
polluting the prefix-cache hash map.
"""
block_size = 8
# Full attn block_size=32, SWA block_size=8, sw=8 -> lcm=32.
# tail = ceil(7/8) = 1; per_segment = 32/8 = 4.
@@ -4226,7 +4214,8 @@ def test_hybrid_cache_blocks_clamped_to_lcm():
Chunks past the last lcm-aligned boundary can never participate in a
cache hit (find_longest_cache_hit always returns lcm-aligned hits), so
caching them only pollutes the prefix-cache hash map and keeps blocks
on the LRU list that could otherwise return to the free pool."""
on the LRU list that could otherwise return to the free pool.
"""
block_size = 16
# Full attn block_size=32, SWA block_size=16 -> lcm=32.
kv_cache_config = KVCacheConfig(
@@ -4361,7 +4350,8 @@ def test_hybrid_local_kv_retention_interval_aligns_in_manager():
)
def test_hybrid_local_kv_retention_interval_rejects_invalid(interval, expected_match):
"""A retention interval that is negative or not a multiple of
scheduler_block_size errors out at construction time."""
scheduler_block_size errors out at construction time.
"""
block_size = 8
kv_cache_config = KVCacheConfig(
num_blocks=100,
@@ -4831,7 +4821,8 @@ def test_block_lookup_cache_multi_blocks_per_key():
def test_can_fit_full_sequence_swa_cap_admits_long_prompt():
"""Hybrid full+SWA model with a pool sized at the startup minimum should
admit a prompt longer than the SWA cap, because SlidingWindowManager
recycles blocks during chunked prefill (issue #39734)."""
recycles blocks during chunked prefill (issue #39734).
"""
block_size = 16
sliding_window = 4 * block_size # 64 tokens
max_num_batched_tokens = 8 * block_size # 128 tokens
@@ -4926,7 +4917,8 @@ def test_full_sequence_admission_reserves_device_blocks_for_immediate_chunk(
def test_can_fit_full_sequence_full_attention_still_gates_oversized():
"""The cap only loosens the SWA group; a prompt that exceeds the
full-attention pool capacity must still be rejected."""
full-attention pool capacity must still be rejected.
"""
block_size = 16
sliding_window = 4 * block_size
max_num_batched_tokens = 8 * block_size
@@ -5045,7 +5037,8 @@ def _take_free_blocks(manager: KVCacheManager, num_blocks: int) -> list[KVCacheB
without removing them. These ref_cnt==0 blocks stand in for evictable
cache-hit blocks left behind by a previous (e.g. preempted) request, and
sitting at the head guarantees a later group's external ``get_new_blocks``
would contend for them on unpatched code (issue #33775)."""
would contend for them on unpatched code (issue #33775).
"""
blocks: list[KVCacheBlock] = []
head = manager.block_pool.free_block_queue.fake_free_list_head
for _ in range(num_blocks):
@@ -5056,7 +5049,8 @@ def _take_free_blocks(manager: KVCacheManager, num_blocks: int) -> list[KVCacheB
def _assert_no_double_allocation(manager: KVCacheManager, req_id: str) -> None:
"""No physical block may be handed out twice across groups, and every
block referenced by the request must have a live ref_cnt."""
block referenced by the request must have a live ref_cnt.
"""
block_ids = manager.get_blocks(req_id).get_block_ids()
flat = [block_id for group in block_ids for block_id in group]
assert len(set(flat)) == len(flat), "Block IDs are not unique across groups"
@@ -5083,7 +5077,8 @@ def _cross_group_cache_hit(
) -> Request:
"""Allocate ``req_id`` with a per-group local prefix hit plus external
(connector) computed tokens, driving the coordinator's two-phase path.
Returns the allocated request so callers can free it (e.g. to preempt)."""
Returns the allocated request so callers can free it (e.g. to preempt).
"""
block_size = _two_phase_block_size(manager)
hit_blocks = _take_free_blocks(manager, num_groups * local_blocks_per_group)
cache_hit = KVCacheBlocks(
@@ -5178,7 +5173,8 @@ def test_swa_free_split_keeps_cached_tail_ahead_of_scratch():
"""Dense retention: freeing an SWA request must place its
uncached scratch blocks at the front of the free queue (recycled first)
and keep its cached checkpoint blocks at the back (retained for prefix
hits). This split is always-on, independent of the retention interval."""
hits). This split is always-on, independent of the retention interval.
"""
block_size = 8
kv_cache_config = KVCacheConfig(
num_blocks=100,
@@ -5289,7 +5285,8 @@ def _make_pure_swa_manager(block_size, sliding_window, num_blocks=100, **kwargs)
def test_pure_swa_retention_interval_caches_sparse_tails():
"""Sparse retention must work for a pure-SWA single-group model, not just
hybrid models: only the per-interval tails plus the latest replay tail are
cached, and a replay still hits the latest replayable boundary."""
cached, and a replay still hits the latest replayable boundary.
"""
block_size = 16
manager = _make_pure_swa_manager(
block_size, sliding_window=block_size, retention_interval=64
@@ -5361,7 +5358,8 @@ def test_pure_swa_retention_latest_only():
def test_pure_swa_dense_retention_caches_all():
"""With retention set to ``None``, a pure-SWA model keeps dense behavior:
every block boundary is a potential hit, so all blocks are cached."""
every block boundary is a potential hit, so all blocks are cached.
"""
block_size = 16
manager = _make_pure_swa_manager(block_size, sliding_window=block_size)
@@ -5390,7 +5388,8 @@ def test_mamba_reachable_block_mask_sparsifies_retention():
"""Mamba state-snapshot retention: with a configured retention interval,
the manager keeps one cached state per interval-sized segment (plus the
latest replay boundary) instead of a snapshot per block, which is what
lets a small attention block_size avoid Mamba dominating the KV pool."""
lets a small attention block_size avoid Mamba dominating the KV pool.
"""
from vllm.v1.core.single_type_kv_cache_manager import MambaManager
block_size = 16
@@ -5428,7 +5427,8 @@ def test_mamba_reachable_block_mask_pins_shared_prefix():
"""A Marconi-detected shared prefix (``shared_prefix_boundary``) lands before
``num_prompt`` so the replay-boundary rule alone would drop it. The mask must
pin the single state block ending on that boundary so sparse retention does
not defeat cross-request shared-prefix reuse."""
not defeat cross-request shared-prefix reuse.
"""
from vllm.v1.core.single_type_kv_cache_manager import MambaManager
block_size = 16
@@ -5476,7 +5476,8 @@ def test_mamba_shared_prefix_survives_zero_retention():
detection) keeps its Mamba state block cached under
``prefix_cache_retention_interval=0``, which otherwise retains only the
end-of-prompt replay boundary. Without this, a shared prefix (junction
before ``num_prompt``) would be recomputed by every sharing request."""
before ``num_prompt``) would be recomputed by every sharing request.
"""
block_size = 16
# 16-block (256-token) prompt; replay boundary is block 240 // 16 - 1 = 14.
@@ -5518,7 +5519,8 @@ def test_mamba_shared_prefix_reuse_under_zero_retention():
detecting request must stay reusable by a later request under
``prefix_cache_retention_interval=0``. Without the pin the junction is
masked out and the later request misses; with it (and under dense) the reuse
is preserved."""
is preserved.
"""
block_size = 16
def last_req_hit(retention, pin):
@@ -5601,7 +5603,8 @@ def test_swa_reachable_block_mask_final_partial_segment():
def test_swa_reachable_block_mask_pins_shared_prefix():
"""SWA analog of the Mamba pin: the shared-prefix junction must keep the
``need``-block sliding-window tail ending on that boundary (not a single
block), so a windowed hit can land there under sparse retention."""
block), so a windowed hit can land there under sparse retention.
"""
from vllm.v1.core.single_type_kv_cache_manager import SlidingWindowManager
block_size = 16
@@ -5646,7 +5649,8 @@ def test_swa_reachable_block_mask_pins_shared_prefix():
def test_swa_reachable_block_mask_with_dcp_scaling():
"""DCP shards each block's KV across ranks, scaling the effective block size.
Verify that dcp_world_size > 1 scales block size in reachability calculations,
producing different masks than dcp_world_size=1."""
producing different masks than dcp_world_size=1.
"""
from vllm.v1.core.single_type_kv_cache_manager import SlidingWindowManager
block_size = 16
@@ -5705,7 +5709,8 @@ def test_swa_reachable_block_mask_sub_block_alignment_is_dense(
multiple of the DCP-scaled block size) cannot be represented exactly. This
happens for hybrid offloading (e.g. Gemma), where alignment_tokens is the
full-attention chunk size and need not divide the SWA block size. The mask
must fall back to dense (None) rather than raise."""
must fall back to dense (None) rather than raise.
"""
from vllm.v1.core.single_type_kv_cache_manager import SlidingWindowManager
spec = SlidingWindowSpec(
@@ -5733,7 +5738,8 @@ def test_mamba_reachable_block_mask_ignores_dcp():
"""Mamba uses TP, not DCP: each rank holds the full recurrent state, so a
state block spans kv_cache_spec.block_size tokens regardless of DCP. The
mask must not scale by dcp_world_size, so retention granularity is
identical for any DCP world size."""
identical for any DCP world size.
"""
from vllm.v1.core.single_type_kv_cache_manager import MambaManager
spec = MambaSpec(
@@ -5767,7 +5773,8 @@ def test_mamba_reachable_block_mask_ignores_dcp():
def test_mamba_reachable_block_mask_large_dcp_stays_sparse():
"""A large DCP world size must not scale the Mamba block size, so it can
never collapse sparse retention into dense caching."""
never collapse sparse retention into dense caching.
"""
from vllm.v1.core.single_type_kv_cache_manager import MambaManager
spec = MambaSpec(
@@ -5802,7 +5809,8 @@ def test_swa_shared_prefix_reuse_under_zero_retention():
"""SWA cross-request analog: a partial shared prefix's sliding-window tail
must stay reusable under ``prefix_cache_retention_interval=0``. Without
the pin the junction window is masked out and a later request misses; with
it (and under dense) reuse is preserved."""
it (and under dense) reuse is preserved.
"""
block_size = 16
def last_req_hit(retention, pin):
+21 -21
View File
@@ -14,10 +14,10 @@ pytestmark = pytest.mark.cpu_test
class TestCheckSequenceRepetition:
"""Unit tests for the check_sequence_repetition function"""
"""Unit tests for the check_sequence_repetition function."""
def test_simple_repetition_detected(self):
"""Test detection of simple repetitive patterns"""
"""Test detection of simple repetitive patterns."""
token_ids = [1, 2, 3, 1, 2, 3, 1, 2, 3]
params = RepetitionDetectionParams(
max_pattern_size=3,
@@ -27,7 +27,7 @@ class TestCheckSequenceRepetition:
assert check_sequence_repetition(token_ids, params)
def test_repetition_below_min_count(self):
"""Test that pattern below min_count is not detected"""
"""Test that pattern below min_count is not detected."""
token_ids = [1, 2, 3, 1, 2, 3]
params = RepetitionDetectionParams(
max_pattern_size=3,
@@ -37,7 +37,7 @@ class TestCheckSequenceRepetition:
assert not check_sequence_repetition(token_ids, params)
def test_two_token_pattern(self):
"""Test detection of 2-token patterns"""
"""Test detection of 2-token patterns."""
token_ids = [1, 2, 1, 2, 1, 2, 1, 2]
params = RepetitionDetectionParams(
max_pattern_size=5,
@@ -47,7 +47,7 @@ class TestCheckSequenceRepetition:
assert check_sequence_repetition(token_ids, params)
def test_no_repetition_varied_sequence(self):
"""Test that non-repetitive sequences are not flagged"""
"""Test that non-repetitive sequences are not flagged."""
token_ids = [1, 2, 3, 4, 5, 6, 7, 8, 9]
params = RepetitionDetectionParams(
max_pattern_size=5,
@@ -57,7 +57,7 @@ class TestCheckSequenceRepetition:
assert not check_sequence_repetition(token_ids, params)
def test_partial_repetition_not_detected(self):
"""Test that incomplete repetitions are not detected"""
"""Test that incomplete repetitions are not detected."""
token_ids = [1, 2, 3, 1, 2, 3, 1, 2, 4]
params = RepetitionDetectionParams(
max_pattern_size=3,
@@ -67,7 +67,7 @@ class TestCheckSequenceRepetition:
assert not check_sequence_repetition(token_ids, params)
def test_empty_token_list(self):
"""Test with empty token list"""
"""Test with empty token list."""
params = RepetitionDetectionParams(
max_pattern_size=3,
min_pattern_size=2,
@@ -76,19 +76,19 @@ class TestCheckSequenceRepetition:
assert not check_sequence_repetition([], params)
def test_detection_disabled_max_size_zero(self):
"""Test that zero max_pattern_size disables detection"""
"""Test that zero max_pattern_size disables detection."""
token_ids = [1, 2, 1, 2, 1, 2]
params = RepetitionDetectionParams()
assert not check_sequence_repetition(token_ids, params)
def test_invalid_min_count(self):
"""Test that min_count < 2 returns False"""
"""Test that min_count < 2 returns False."""
token_ids = [1, 2, 1, 2]
params = RepetitionDetectionParams()
assert not check_sequence_repetition(token_ids, params)
def test_repetition_at_end_of_sequence(self):
"""Test detection when repetition occurs at the end"""
"""Test detection when repetition occurs at the end."""
token_ids = [1, 2, 3, 4, 5, 6, 5, 6, 5, 6]
params = RepetitionDetectionParams(
max_pattern_size=3,
@@ -98,7 +98,7 @@ class TestCheckSequenceRepetition:
assert check_sequence_repetition(token_ids, params)
def test_large_pattern_many_repetitions(self):
"""Test large pattern repeated many times"""
"""Test large pattern repeated many times."""
token_ids = [1, 2, 3, 4, 5, 6, 7, 8] * 5
params = RepetitionDetectionParams(
max_pattern_size=10,
@@ -114,10 +114,10 @@ class TestCheckSequenceRepetition:
class TestRepetitionDetectionIntegration:
"""Integration tests for repetition detection in check_stop"""
"""Integration tests for repetition detection in check_stop."""
def test_basic_repetition_stops_generation(self):
"""Test that repetition is detected and stops generation"""
"""Test that repetition is detected and stops generation."""
params = SamplingParams(
max_tokens=100,
repetition_detection=RepetitionDetectionParams(
@@ -138,7 +138,7 @@ class TestRepetitionDetectionIntegration:
assert request.stop_reason == "repetition_detected"
def test_detection_disabled_no_stop(self):
"""Test that disabled detection doesn't stop generation"""
"""Test that disabled detection doesn't stop generation."""
params = SamplingParams(
max_tokens=100,
)
@@ -152,7 +152,7 @@ class TestRepetitionDetectionIntegration:
assert not check_stop(request, max_model_len=1024)
def test_repetition_respects_min_tokens(self):
"""Test that repetition detection respects min_tokens"""
"""Test that repetition detection respects min_tokens."""
params = SamplingParams(
min_tokens=10,
max_tokens=100,
@@ -172,7 +172,7 @@ class TestRepetitionDetectionIntegration:
assert not check_stop(request, max_model_len=1024)
def test_no_repetition_continues_generation(self):
"""Test that non-repetitive tokens don't stop generation"""
"""Test that non-repetitive tokens don't stop generation."""
params = SamplingParams(
max_tokens=100,
repetition_detection=RepetitionDetectionParams(
@@ -191,7 +191,7 @@ class TestRepetitionDetectionIntegration:
assert not check_stop(request, max_model_len=1024)
def test_pattern_at_size_boundary(self):
"""Test detection at exact pattern size boundary"""
"""Test detection at exact pattern size boundary."""
params = SamplingParams(
max_tokens=100,
repetition_detection=RepetitionDetectionParams(
@@ -211,7 +211,7 @@ class TestRepetitionDetectionIntegration:
assert request.status == RequestStatus.FINISHED_REPETITION
def test_multiple_pattern_sizes_checked(self):
"""Test that function checks pattern sizes in range"""
"""Test that function checks pattern sizes in range."""
params = SamplingParams(
max_tokens=100,
repetition_detection=RepetitionDetectionParams(
@@ -231,7 +231,7 @@ class TestRepetitionDetectionIntegration:
assert request.status == RequestStatus.FINISHED_REPETITION
def test_eos_takes_precedence_over_repetition(self):
"""Test that EOS token stops before repetition check"""
"""Test that EOS token stops before repetition check."""
params = SamplingParams(
max_tokens=100,
stop_token_ids=[999],
@@ -252,7 +252,7 @@ class TestRepetitionDetectionIntegration:
assert request.status == RequestStatus.FINISHED_STOPPED
def test_min_pattern_size_filters_small_patterns(self):
"""Test that min_pattern_size filters out smaller patterns"""
"""Test that min_pattern_size filters out smaller patterns."""
params = SamplingParams(
max_tokens=100,
repetition_detection=RepetitionDetectionParams(
@@ -271,7 +271,7 @@ class TestRepetitionDetectionIntegration:
assert not check_stop(request, max_model_len=1024)
def test_high_repetition_threshold(self):
"""Test that high min_count requires many repetitions"""
"""Test that high min_count requires many repetitions."""
params = SamplingParams(
max_tokens=100,
repetition_detection=RepetitionDetectionParams(
+66 -54
View File
@@ -295,7 +295,7 @@ def test_schedule_multimodal_requests():
def test_async_scheduling_pp_allows_rescheduling_with_output_placeholders():
"""Async scheduling + PP: allow multi-step in-flight scheduling per request"""
"""Async scheduling + PP: allow multi-step in-flight scheduling per request."""
scheduler = create_scheduler(async_scheduling=True, pipeline_parallel_size=2)
(req,) = create_requests(num_requests=1, num_tokens=8)
scheduler.add_request(req)
@@ -538,7 +538,8 @@ def test_throttle_defers_inflight_prefill_chunk():
"""DP prefill balancing throttles ALL prefill compute on a throttled step,
not just new admissions: an in-progress (chunked) prefill already in the
running queue is also deferred, so the step runs decode-only, while a
separate decode keeps being scheduled."""
separate decode keeps being scheduled.
"""
scheduler = create_scheduler(
max_num_seqs=16, max_num_batched_tokens=50, enable_chunked_prefill=True
)
@@ -592,7 +593,8 @@ def test_throttle_capacity_bound_guard_admits():
"""Saturation guard: if a cadence-aligned release step cannot drain the
waiting prefill queue (it ran out of token budget), the throttle backs off on
the next step so the backlog cannot grow into a TTFT avalanche -- prefills are
admitted even though throttle_prefills is set."""
admitted even though throttle_prefills is set.
"""
scheduler = create_scheduler(
max_num_seqs=16, max_num_batched_tokens=200, enable_chunked_prefill=True
)
@@ -785,7 +787,7 @@ def test_update_from_output_routes_sampling_masks_by_request():
def test_stop_via_update_from_output():
"""Test stopping behavior through update_from_output"""
"""Test stopping behavior through update_from_output."""
scheduler = create_scheduler(num_speculative_tokens=1)
# Test case 1: Stop on EOS token
@@ -1291,7 +1293,8 @@ def test_pending_hisparse_reclamation_defers_preemption(monkeypatch):
def test_prefix_cache_query_not_inflated_by_connector_defer():
"""The GPU prefix-cache query is recorded at admission, so a request the
connector defers several times is counted once, not once per retry."""
connector defers several times is counted once, not once per retry.
"""
num_defers_before_matching = 3
scheduler = create_scheduler(
enable_prefix_caching=True,
@@ -1319,7 +1322,8 @@ def test_prefix_cache_query_not_inflated_by_connector_defer():
def test_preemption_re_records_prefix_cache_query():
"""A preempted request re-enters the lookup on resume, so its recomputation
is counted again into the preempted stats."""
is counted again into the preempted stats.
"""
scheduler = create_scheduler(enable_prefix_caching=True)
request = create_requests(num_requests=1)[0]
scheduler.add_request(request)
@@ -1355,7 +1359,8 @@ def test_preemption_re_records_prefix_cache_query():
def test_prefix_cache_stats_not_recorded_when_caching_disabled():
"""With prefix caching off there is no local lookup, so admitting a request
records no phantom miss."""
records no phantom miss.
"""
scheduler = create_scheduler(enable_prefix_caching=False)
for request in create_requests(num_requests=2):
scheduler.add_request(request)
@@ -1369,7 +1374,8 @@ def test_prefix_cache_stats_not_recorded_when_caching_disabled():
def test_prefix_cache_stats_counted_once_for_retried_then_scheduled_request():
"""A real cache hit rejected once by allocate_slots and admitted on the next
step is counted exactly once, hits included."""
step is counted exactly once, hits included.
"""
block_size = 16
scheduler = create_scheduler(
enable_prefix_caching=True,
@@ -2130,7 +2136,6 @@ def _assert_right_scheduler_output(
expected_num_scheduled_tokens: int,
):
"""Check if SchedulerOutput is correct after remote KV cache hit."""
# We should inject the kv_connector_metadata.
assert len(output.kv_connector_metadata.requests) == num_requests
@@ -2148,7 +2153,6 @@ def _assert_right_kv_cache_manager(
num_total_blocks: int,
):
"""Check whether KVCacheManager is correct after allocate."""
# Make sure the request stats are right.
EXPECTED_TOTAL_BLOCKS = num_tokens // block_size
for req in requests:
@@ -2179,7 +2183,6 @@ def _step_until_done(
model_runner_output: ModelRunnerOutput,
):
"""Loop over schedule(), update_from_output() until finished."""
all_finished = False
_ = scheduler.update_from_output(output, model_runner_output)
while not all_finished:
@@ -2207,7 +2210,6 @@ def _num_waiting_requests(scheduler: Scheduler) -> int:
def _step_until_kv_transfer_finished(scheduler: Scheduler, req_ids: list[str]):
"""Cycle requests through a KV transfer cycle."""
# Requests should first transition to WAITING_FOR_REMOTE_KVS
output = scheduler.schedule()
assert _num_waiting_requests(scheduler) == len(req_ids)
@@ -2291,11 +2293,9 @@ def test_has_sync_kv_loads(
@pytest.mark.parametrize("is_async", [False, True])
def test_kv_connector_basic(is_async: bool):
"""
Test whether Scheduler with KVConnector schedules tokens, allocates
"""Test whether Scheduler with KVConnector schedules tokens, allocates
memory, and cleans up requests as expected under normal operation.
"""
# Setup Scheduler.
BLOCK_SIZE = 16
NUM_MATCHED_NEW_TOKENS = BLOCK_SIZE * 2
@@ -2420,11 +2420,9 @@ def test_kv_connector_basic(is_async: bool):
@pytest.mark.parametrize("is_async", [False, True])
@pytest.mark.parametrize("local_cache_hits", [False, True])
def test_external_prefix_cache_metrics(is_async: bool, local_cache_hits: bool):
"""
Verify connector prefix cache metrics are updated
"""Verify connector prefix cache metrics are updated
correctly when the scheduler processes requests with KV connector hits.
"""
BLOCK_SIZE = 16
if local_cache_hits:
NUM_MATCHED_NEW_TOKENS = BLOCK_SIZE * 2 # 32 tokens
@@ -2540,11 +2538,9 @@ def test_external_prefix_cache_metrics(is_async: bool, local_cache_hits: bool):
"use_ec_connector, ec_role", [(False, None), (True, "ec_consumer")]
)
def test_kv_connector_unable_to_allocate(use_ec_connector, ec_role):
"""
Test whether scheduler with KVConnector is able to handle
"""Test whether scheduler with KVConnector is able to handle
unable to allocate (run out of blocks in allocate_slots().
"""
# Setup Scheduler With Mock External Cache Hit.
BLOCK_SIZE = 4
NUM_BLOCKS = 10
@@ -2627,11 +2623,9 @@ def test_kv_connector_unable_to_allocate(use_ec_connector, ec_role):
def test_kv_connector_handles_preemption(
is_async, use_ec_connector, ec_role, use_v2_model_runner
):
"""
Test whether scheduler with KVConnector is able to handle
"""Test whether scheduler with KVConnector is able to handle
unable to allocate (run out of blocks in allocate_slots().
"""
# Setup Scheduler With Mock External Cache Hit.
BLOCK_SIZE = 2
# NOTE: there is 1 null block, so this is 6 blocks.
@@ -2842,7 +2836,6 @@ def assert_scheduler_empty(scheduler: Scheduler):
def test_memory_leak():
"""Test that we do not have a memory leak."""
scheduler = create_scheduler(enable_prefix_caching=True)
NUM_REQUESTS = 5
@@ -2899,6 +2892,7 @@ def create_scheduler_with_priority(
Returns:
{class}`Scheduler` instance with priority scheduling
"""
model_config = ModelConfig(
model=model,
@@ -3100,7 +3094,8 @@ def create_requests_with_priority(
def test_priority_scheduling_basic_ordering():
"""Test that requests are scheduled in priority order
(lower value = higher priority)."""
(lower value = higher priority).
"""
scheduler = create_scheduler_with_priority()
# Create requests with different priorities
@@ -3129,7 +3124,8 @@ def test_priority_scheduling_basic_ordering():
def test_priority_scheduling_arrival_time_tiebreaker():
"""Test that arrival time is used
as tiebreaker when priorities are equal."""
as tiebreaker when priorities are equal.
"""
scheduler = create_scheduler_with_priority()
# Create requests with same priority but different arrival times
@@ -3286,7 +3282,8 @@ def test_priority_scheduling_preemption():
def test_priority_scheduling_no_preemption_when_space_available():
"""Test that preemption doesn't happen
when there's space for new requests."""
when there's space for new requests.
"""
scheduler = create_scheduler_with_priority(
max_num_seqs=3, # Allow 3 concurrent requests
max_num_batched_tokens=200, # Sufficient token budget
@@ -3339,7 +3336,8 @@ def test_priority_scheduling_no_preemption_when_space_available():
def test_priority_scheduling_preemption_victim_selection():
"""Test that the correct victim is selected for
preemption based on priority and arrival time."""
preemption based on priority and arrival time.
"""
# This test verifies the priority-based victim selection logic
# by checking the waiting queue order after adding requests with different
# priorities
@@ -3459,7 +3457,8 @@ def test_priority_scheduling_waiting_queue_order():
def test_priority_scheduling_fcfs_fallback():
"""Test that FCFS behavior is maintained when all
requests have same priority."""
requests have same priority.
"""
scheduler = create_scheduler_with_priority()
# Create requests with same priority but different arrival times
@@ -3528,7 +3527,8 @@ def test_priority_scheduling_with_limited_slots():
def test_priority_scheduling_heap_property():
"""Test that the waiting queue maintains heap
property for priority scheduling."""
property for priority scheduling.
"""
scheduler = create_scheduler_with_priority(
max_num_seqs=1, # Only one request can run at a time
)
@@ -3753,7 +3753,8 @@ def test_priority_scheduling_preemption_and_resumption_when_out_of_kv(
use_ec_connector, ec_role, use_v2_model_runner
):
"""Test that priority scheduling preempts lower priority requests
when out of KV cache space."""
when out of KV cache space.
"""
# Create scheduler with very limited memory to force preemption
scheduler = create_scheduler_with_priority(
max_num_seqs=2, # Allow multiple requests
@@ -3898,7 +3899,8 @@ def test_chunked_prefill_disabled_for_encoder_decoder(
enable_chunked_prefill: bool, is_encoder_decoder: bool, expect_enabled: bool
) -> None:
"""Validate that chunked prefill is appropriately disabled for
encoder-decoder models."""
encoder-decoder models.
"""
scheduler_config = SchedulerConfig(
enable_chunked_prefill=enable_chunked_prefill,
is_encoder_decoder=is_encoder_decoder,
@@ -3927,7 +3929,8 @@ def _validate_chunked_prefill_settings_for_encoder_decoder(
scheduler_config: SchedulerConfig, is_encoder_decoder: bool, expect_enabled: bool
) -> None:
"""Validate chunked prefill settings in the scheduler config for
encoder-decoder models."""
encoder-decoder models.
"""
assert scheduler_config.enable_chunked_prefill is expect_enabled
if is_encoder_decoder:
# Encoder-decoder models should automatically disable chunked multimodal
@@ -3987,7 +3990,7 @@ def _assert_right_ec_connector_metadata(
output: SchedulerOutput,
mm_features_list: list[MultiModalFeatureSpec],
):
"""Verify that ECConnector metadata EXACTLY matches the input MM data"""
"""Verify that ECConnector metadata EXACTLY matches the input MM data."""
# Get the connector metadata
metadata = output.ec_connector_metadata
@@ -4017,8 +4020,8 @@ def _assert_right_encoder_inputs(
):
"""Verify that requests/mm_hashes should (not) in scheduled encoder input
If check_exist is False, this function returns True
if requests are NOT in encoder inputs"""
if requests are NOT in encoder inputs
"""
# Get the scheduled encoder inputs
# NOTE: scheduled_encoder_inputs is a dictionary with request id as key
scheduled_encoder_inputs = output.scheduled_encoder_inputs
@@ -4179,7 +4182,8 @@ def test_ec_connector_text_only_request(use_kv_connector):
@pytest.mark.parametrize("use_kv_connector", [False, True])
def test_ec_connector_cache_hit_external_load(use_kv_connector):
"""Test ec_consumer loads from external cache when hit.
A normal basic operation for EPD disaggrgation"""
A normal basic operation for EPD disaggrgation
"""
scheduler = create_scheduler(
model="llava-hf/llava-1.5-7b-hf",
enable_prefix_caching=True,
@@ -4527,11 +4531,9 @@ def test_ec_connector_schedule_multiple_requests(cache_exist, use_kv_connector):
@pytest.mark.parametrize("use_kv_connector", [False, True])
def test_ec_connector_unable_to_allocate(use_kv_connector):
"""
Test whether scheduler with ECConnector is able to handle
"""Test whether scheduler with ECConnector is able to handle
unable to allocate (run out of blocks).
"""
# Setup Scheduler With Mock External Cache Hit.
BLOCK_SIZE = 4
NUM_BLOCKS = 10
@@ -4628,7 +4630,8 @@ def test_priority_scheduling_ec_connector_preemption_and_resumption(
cache_exist, use_kv_connector
):
"""Test that priority scheduling preempts lower priority requests
when out of KV cache space."""
when out of KV cache space.
"""
# Create scheduler with very limited memory to force preemption
scheduler = create_scheduler_with_priority(
model="llava-hf/llava-1.5-7b-hf",
@@ -4860,8 +4863,7 @@ def test_priority_scheduling_ec_connector_preemption_and_resumption(
@pytest.mark.parametrize("use_kv_connector", [False, True])
def test_ec_connector_allocate_encoder_tokens_with_external_load(use_kv_connector):
"""
Scenario:
"""Scenario:
- Encoder cache size: 32
- Request A: 1 feature (12 tokens) → NOT cached remotely.
- Request B: 3 features (3 x 10 tokens) → ALL cached remotely.
@@ -5298,7 +5300,8 @@ def test_scheduler_kv_connector_stats():
def test_ec_connector_update_connector_output_called():
"""Test that worker-side EC connector output is forwarded to the
EC connector's update_connector_output hook."""
EC connector's update_connector_output hook.
"""
scheduler = create_scheduler(
model="llava-hf/llava-1.5-7b-hf",
use_ec_connector=True,
@@ -5503,7 +5506,8 @@ def test_variable_length_cross_attn_block_allocation():
def test_cross_attn_blocks_not_over_allocated():
"""Test that cross-attention blocks are not over-allocated compared to
what each request actually needs."""
what each request actually needs.
"""
from math import ceil
block_size = 16
@@ -5599,7 +5603,8 @@ def test_cross_attn_blocks_not_under_allocated():
def test_cross_attn_zero_blocks_without_encoder_inputs():
"""Test that requests without encoder inputs get zero cross-attention
blocks, even when the scheduler is configured for encoder-decoder."""
blocks, even when the scheduler is configured for encoder-decoder.
"""
block_size = 16
scheduler = _create_encoder_decoder_scheduler(block_size=block_size)
@@ -5704,7 +5709,8 @@ def test_free_encoder_inputs_respects_unconfirmed_placeholders():
evicted entry and crashes the engine with "Encoder cache miss". The
scheduler must retain the input until the *confirmed* progress
(num_computed_tokens - num_output_placeholders) passes the range end, so
that no pending rejection can rewind into the range."""
that no pending rejection can rewind into the range.
"""
scheduler = create_scheduler(
model="llava-hf/llava-1.5-7b-hf",
num_speculative_tokens=3,
@@ -5815,7 +5821,8 @@ def test_free_encoder_inputs_defers_for_eagle_lookahead():
position so the drafter's +1 look-ahead mm-embedding gather (which reads one
position past the target's computed range) still finds it cached. This is
the primary mechanism that prevents the drafter "Encoder cache miss"; the
worker-side token-embedding fallback is only a backstop."""
worker-side token-embedding fallback is only a backstop.
"""
scheduler = create_scheduler(model="llava-hf/llava-1.5-7b-hf")
# create_scheduler only builds ngram spec configs; force the eagle path that
# _free_encoder_inputs keys off (its read-ahead deferral).
@@ -5845,7 +5852,8 @@ def test_free_encoder_inputs_defers_for_eagle_lookahead():
def test_free_encoder_inputs_unchanged_without_spec_decode():
"""Without speculative decoding, encoder inputs are freed as soon as
num_computed_tokens passes the placeholder range, as before."""
num_computed_tokens passes the placeholder range, as before.
"""
scheduler = create_scheduler(model="llava-hf/llava-1.5-7b-hf")
mm_positions = [[PlaceholderRange(offset=50, length=100)]]
request = create_requests(
@@ -5876,7 +5884,8 @@ def test_encoder_cache_retained_across_preemption_and_resume():
encoder input must pull the still-cached entry back out of `freeable`
without scheduling a recompute, keeping the scheduler and worker
consistent. The spec-rollback retention margin does not gate this path,
so it is covered separately here."""
so it is covered separately here.
"""
scheduler = create_scheduler(
model="llava-hf/llava-1.5-7b-hf",
num_speculative_tokens=3,
@@ -5920,7 +5929,8 @@ def test_encoder_cache_recomputed_when_evicted_during_preemption():
pressure before it resumes, the scheduler reports the mm_hash as freed
(so the worker drops it) and a resume must schedule a recompute rather
than assume the worker still holds it. check_and_update_cache must
return False so the encoder input is re-scheduled."""
return False so the encoder input is re-scheduled.
"""
scheduler = create_scheduler(
model="llava-hf/llava-1.5-7b-hf",
num_speculative_tokens=3,
@@ -6571,7 +6581,8 @@ def _decode_ready_request(scheduler):
def test_update_draft_token_ids_strips_ngram_padding():
"""ngram_gpu pads unfilled draft slots with -1; manager validation must
strip them before grammar.validate_tokens (which otherwise raises)."""
strip them before grammar.validate_tokens (which otherwise raises).
"""
scheduler = create_scheduler(num_speculative_tokens=4)
request = _decode_ready_request(scheduler)
@@ -6590,7 +6601,8 @@ def test_update_draft_token_ids_strips_ngram_padding():
def test_update_draft_token_ids_in_output_strips_padding():
"""Same guard on the output path; the -1 pad-back for the rejected count
is preserved (only the input to manager validation is stripped)."""
is preserved (only the input to manager validation is stripped).
"""
scheduler = create_scheduler(num_speculative_tokens=4)
request = _decode_ready_request(scheduler)
@@ -97,7 +97,8 @@ def test_num_in_flight_tokens_accounting():
def test_swa_free_waits_for_in_flight_step():
"""Async: out-of-window blocks stay allocated until the step that still
reads them has been processed."""
reads them has been processed.
"""
scheduler = _create_swa_scheduler(async_scheduling=True)
request = create_requests(
num_requests=1, num_tokens=NUM_PROMPT_TOKENS, block_size=BLOCK_SIZE
@@ -132,7 +133,8 @@ def test_swa_free_waits_for_in_flight_step():
def test_swa_free_immediate_when_sync():
"""Sync: no in-flight step at schedule time, frees happen at the first
decode allocation as before."""
decode allocation as before.
"""
scheduler = _create_swa_scheduler(async_scheduling=False)
request = create_requests(
num_requests=1, num_tokens=NUM_PROMPT_TOKENS, block_size=BLOCK_SIZE
@@ -171,7 +173,8 @@ def test_swa_admission_cap_accounts_for_overlapping_batches():
def test_chunked_local_free_waits_for_in_flight_step():
"""Chunked-local attention frees whole chunks left of the current one, and
is exposed to the same load-WAR: with async scheduling those chunks must
stay allocated until the in-flight step that still reads them settles."""
stay allocated until the in-flight step that still reads them settles.
"""
scheduler = _create_chunked_scheduler(async_scheduling=True)
request = create_requests(
num_requests=1, num_tokens=NUM_PROMPT_TOKENS, block_size=BLOCK_SIZE
@@ -226,7 +229,8 @@ def test_connector_finish_frees_on_settled_basis():
is handed to a KV connector (simple CPU offload / NIXL store), must use the
same processed-token basis. Otherwise the connector reads/hands off a block
the still-in-flight step is optimistically counted as done with, which is
the load-path WAR this fix closes."""
the load-path WAR this fix closes.
"""
scheduler = create_scheduler(
block_size=BLOCK_SIZE,
async_scheduling=True,
+1
View File
@@ -89,6 +89,7 @@ def create_scheduler(
Returns:
{class}`Scheduler` instance
"""
model_config = ModelConfig(
model=model,