Add pydocstyle to the ruff rules (#52136)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: Harry Mellor <hej.mellor@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
804e5377aa
commit
8c1557a79c
@@ -2,7 +2,8 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Fine-grained partial prefix-cache hits for hybrid (full attention + mamba
|
||||
"align") models: scheduler chunk splitting, partial tail registration, CoW
|
||||
on partial hits, and same-step deferral."""
|
||||
on partial hits, and same-step deferral.
|
||||
"""
|
||||
|
||||
from math import lcm
|
||||
from types import SimpleNamespace
|
||||
@@ -174,7 +175,8 @@ def test_mamba_align_split_partial_tail_schedule(dcp_world_size: int):
|
||||
"""Chunk ends with partial hits on: block-aligned chunks, one extra stop
|
||||
at the prompt's last hash boundary (registering the partial tail), then
|
||||
the remaining tokens. block=512, hash=32, prompt=10000, budget=8192:
|
||||
0 -> 8192 -> 9728 -> 9984 -> 10000."""
|
||||
0 -> 8192 -> 9728 -> 9984 -> 10000.
|
||||
"""
|
||||
block_size = 512
|
||||
scheduler_block_size = block_size * dcp_world_size
|
||||
hash_block_size = 32
|
||||
@@ -742,7 +744,8 @@ def test_hash_aligned_query_end_uses_regular_partial_tail():
|
||||
|
||||
def test_external_mamba_hit_same_block_uses_running_cow_on_continue():
|
||||
"""An external mid-block hit must become a running request even when its
|
||||
first continuation does not need another Mamba block."""
|
||||
first continuation does not need another Mamba block.
|
||||
"""
|
||||
hash_block_size = 2
|
||||
mamba_block_size = 4 * hash_block_size
|
||||
kv_cache_config = KVCacheConfig(
|
||||
@@ -814,7 +817,8 @@ def test_external_mamba_hit_same_block_uses_running_cow_on_continue():
|
||||
|
||||
def test_boundary_state_offloads_returns_cow_target():
|
||||
"""Boundary hand-offs expose aligned snapshots and the partial-tail CoW
|
||||
target, never the overwritten CoW source."""
|
||||
target, never the overwritten CoW source.
|
||||
"""
|
||||
hash_block_size = 2
|
||||
block_size = 2 * hash_block_size
|
||||
kv_cache_config = KVCacheConfig(
|
||||
@@ -1000,7 +1004,8 @@ def test_connector_finish_registers_partial_tail_before_cleanup():
|
||||
def test_boundary_state_offload_dropped_when_request_freed_before_drain():
|
||||
"""A hand-off recorded in the same scheduling pass as the request's death
|
||||
must not be drained: its release hook has already run, so draining would
|
||||
leak a pinned block."""
|
||||
leak a pinned block.
|
||||
"""
|
||||
hash_block_size = 2
|
||||
block_size = 2 * hash_block_size
|
||||
kv_cache_config = KVCacheConfig(
|
||||
@@ -1047,7 +1052,8 @@ def test_boundary_state_offload_dropped_when_request_freed_before_drain():
|
||||
|
||||
def test_boundary_state_offloads_block_aligned_prompt():
|
||||
"""A prompt ending on a block boundary registers no CoW partial tail; its
|
||||
boundary state block is handed off as a snapshot instead (once)."""
|
||||
boundary state block is handed off as a snapshot instead (once).
|
||||
"""
|
||||
hash_block_size = 2
|
||||
block_size = 2 * hash_block_size
|
||||
kv_cache_config = KVCacheConfig(
|
||||
@@ -1101,7 +1107,8 @@ def test_boundary_state_offloads_block_aligned_prompt():
|
||||
def test_truncate_computed_blocks_preserves_sparse_prefix_positions():
|
||||
"""truncate_computed_blocks slices each group by its own block size,
|
||||
keeps null placeholders in the retained prefix, and leaves the original
|
||||
lookup result untouched (pure view, no refcount changes)."""
|
||||
lookup result untouched (pure view, no refcount changes).
|
||||
"""
|
||||
hash_block_size = 2
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=24,
|
||||
@@ -1156,7 +1163,8 @@ def test_truncate_computed_blocks_preserves_sparse_prefix_positions():
|
||||
|
||||
def test_truncate_computed_blocks_allows_short_mamba_group_only():
|
||||
"""External state may replace a short Mamba hit, but other groups must
|
||||
cover the aligned local endpoint."""
|
||||
cover the aligned local endpoint.
|
||||
"""
|
||||
hash_block_size = 2
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=24,
|
||||
@@ -1298,7 +1306,8 @@ def test_hybrid_mamba_partial_tail_owner_continue_preserves_later_hit():
|
||||
def test_hybrid_mamba_moved_partial_entry_defers_same_step_hit():
|
||||
"""The owner's move re-arms the same-step guard: the moved entry is
|
||||
filled by this step's copy, and chained same-step copies read stale
|
||||
sources, so a request hitting it in the move step must be deferred."""
|
||||
sources, so a request hitting it in the move step must be deferred.
|
||||
"""
|
||||
hash_block_size = 2
|
||||
block_size = 2 * hash_block_size
|
||||
kv_cache_config = KVCacheConfig(
|
||||
@@ -1588,7 +1597,8 @@ def test_hybrid_partial_hash_truncates_full_attention_hit_length():
|
||||
|
||||
def test_cow_retained_blocks_returned_for_release():
|
||||
"""new_step_starts returns the CoW copy retentions instead of freeing
|
||||
them; the scheduler owns releasing them once the copy has run."""
|
||||
them; the scheduler owns releasing them once the copy has run.
|
||||
"""
|
||||
hash_block_size = 2
|
||||
block_size = 2 * hash_block_size
|
||||
kv_cache_config = KVCacheConfig(
|
||||
@@ -1641,7 +1651,8 @@ def test_cow_retained_blocks_returned_for_release():
|
||||
|
||||
def test_free_cow_retained_blocks_defers_until_copy_step_processed():
|
||||
"""Scheduler releases CoW retentions immediately when the copy's step has
|
||||
been processed (or deferral is off), and defers them otherwise."""
|
||||
been processed (or deferral is off), and defers them otherwise.
|
||||
"""
|
||||
from collections import deque
|
||||
|
||||
freed: list = []
|
||||
@@ -1678,7 +1689,8 @@ def test_free_cow_retained_blocks_defers_until_copy_step_processed():
|
||||
def test_full_attention_eagle_drops_one_hash_unit():
|
||||
"""With fine-grained partial hits, eagle rewinds the hit by one hash unit
|
||||
instead of a whole cache block: the tail block's KV is append-only, so it
|
||||
still covers the reduced length and stays in the hit as a partial block."""
|
||||
still covers the reduced length and stays in the hit as a partial block.
|
||||
"""
|
||||
from vllm.v1.core.block_pool import BlockPool
|
||||
from vllm.v1.core.single_type_kv_cache_manager import FullAttentionManager
|
||||
|
||||
@@ -1754,7 +1766,8 @@ def test_hybrid_partial_hit_with_eagle_stays_within_group_blocks():
|
||||
"""Regression: with eagle, the mamba group must not receive the eagle
|
||||
lookup margin — its finder never applies the drop, so it could return a
|
||||
hit past the blocks the (dropped) full-attention group covers, crashing
|
||||
the consumer's CoW with block_idx >= len(req_blocks)."""
|
||||
the consumer's CoW with block_idx >= len(req_blocks).
|
||||
"""
|
||||
hash_block_size = 2
|
||||
block_size = 2 * hash_block_size
|
||||
kv_cache_config = KVCacheConfig(
|
||||
@@ -2356,7 +2369,8 @@ def test_retention_snapshots_handed_off_with_exact_block_ids():
|
||||
"""Under sparse retention, each retained mamba boundary state block is
|
||||
handed to the connector with its exact block id. Connectors must not resolve
|
||||
align-mode state blocks positionally because the block table is not
|
||||
append-only."""
|
||||
append-only.
|
||||
"""
|
||||
hash_block_size = 2
|
||||
block_size = 4
|
||||
manager = make_kv_cache_manager(
|
||||
@@ -2397,7 +2411,8 @@ def test_snapshot_handoff_dense_default_retention():
|
||||
"""With dense (default) retention, every materialized mamba boundary
|
||||
state block is handed off — regular mamba-align + prefix-match-unit
|
||||
deployments get store-able boundary snapshots without setting a
|
||||
prefix-cache retention interval."""
|
||||
prefix-cache retention interval.
|
||||
"""
|
||||
hash_block_size = 2
|
||||
block_size = 4
|
||||
manager = make_kv_cache_manager(
|
||||
@@ -2429,7 +2444,8 @@ def test_snapshot_handoff_dense_default_retention():
|
||||
|
||||
def _run_chunked_prefill(manager, req, chunk_size, num_chunks):
|
||||
"""Prefill ``req`` one ``chunk_size`` chunk per scheduler step, yielding the
|
||||
boundary-state hand-offs offered (and claimed) at each step."""
|
||||
boundary-state hand-offs offered (and claimed) at each step.
|
||||
"""
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req)
|
||||
for step in range(num_chunks):
|
||||
manager.new_step_starts()
|
||||
@@ -2493,7 +2509,8 @@ def test_boundary_states_offered_past_prompt_for_resumed_prefill():
|
||||
``num_prompt_tokens``. A resumed request re-prefills its generated tokens
|
||||
and every group re-saves them, so filtering on the original prompt length
|
||||
would silently strip the mamba key for boundaries full attention still
|
||||
stores; only the connector knows where its save window ends."""
|
||||
stores; only the connector knows where its save window ends.
|
||||
"""
|
||||
block_size = 8
|
||||
prompt_len = block_size
|
||||
manager = make_kv_cache_manager(
|
||||
|
||||
@@ -552,7 +552,8 @@ def _create_async_pp_scheduler(
|
||||
|
||||
def _assert_ordered_subset(delivered: list[int], emitted: list[int]) -> None:
|
||||
"""Delivered tokens must be an order-preserving subset of the emitted
|
||||
tokens with no duplicates (tokens are globally unique)."""
|
||||
tokens with no duplicates (tokens are globally unique).
|
||||
"""
|
||||
it = iter(emitted)
|
||||
for token in delivered:
|
||||
assert token in it, f"token {token} delivered out of order or twice"
|
||||
@@ -562,7 +563,8 @@ def _assert_positions_consistent(req, engine: PipelinedEngine) -> None:
|
||||
"""The i-th delivered output token must be one the runner sampled for
|
||||
exactly sequence position prompt_len + i: catches a preempted request's
|
||||
stale output landing on a position the resumed request resampled (or
|
||||
vice versa), which token-stream equality alone cannot see."""
|
||||
vice versa), which token-stream equality alone cannot see.
|
||||
"""
|
||||
for i, token in enumerate(req.output_token_ids):
|
||||
expected = req.num_prompt_tokens + i
|
||||
actual = engine.emitted_position[token]
|
||||
|
||||
@@ -178,7 +178,8 @@ def _shared_layout_config():
|
||||
class TestCSALinearGrouping:
|
||||
"""A CSA + linear-attention model (sparse attention with a compressor ring,
|
||||
plus sharded GDN and one TP-replicated PLE state) goes through the generic
|
||||
packed-group path; no model-specific branch is involved."""
|
||||
packed-group path; no model-specific branch is involved.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _mamba_groups(groups):
|
||||
@@ -193,7 +194,8 @@ class TestCSALinearGrouping:
|
||||
def test_replicated_state_gets_its_own_group(self):
|
||||
"""The PLE state is TP-replicated and a different size, so it must not
|
||||
share a manager group with the sharded GDN states (the NIXL worker
|
||||
requires exactly one single-layer replicated group)."""
|
||||
requires exactly one single-layer replicated group).
|
||||
"""
|
||||
groups = get_kv_cache_groups(_shared_layout_config(), _make_csa_linear_specs())
|
||||
|
||||
mamba_groups = self._mamba_groups(groups)
|
||||
@@ -246,7 +248,8 @@ class TestCSALinearGrouping:
|
||||
def test_scratch_group_survives_computed_block_truncation(self):
|
||||
"""The scratch group contributes no computed blocks, so truncating a
|
||||
lookup result must skip it: its block size is the ring capacity, which
|
||||
neither divides the hit length nor bounds the (empty) block list."""
|
||||
neither divides the hit length nor bounds the (empty) block list.
|
||||
"""
|
||||
config = _shared_layout_config()
|
||||
config.cache_config.enable_prefix_caching = True
|
||||
groups = get_kv_cache_groups(config, _make_csa_linear_specs())
|
||||
@@ -360,7 +363,8 @@ class TestCSALinearGrouping:
|
||||
per page size, which is emitted whole, or a single-layer state that
|
||||
cannot be split at all -- the mamba layers must be sized against it.
|
||||
Sizing them against a narrower bucket splits them past what the block
|
||||
already fits, spending a pool block per extra group for no saving."""
|
||||
already fits, spending a pool block per extra group for no saving.
|
||||
"""
|
||||
config = _mock_vllm_config("BLNHC")
|
||||
config.speculative_config = None
|
||||
specs = {}
|
||||
@@ -405,7 +409,8 @@ class TestSlidingWindowBucketCap:
|
||||
unbalanced 8-layer paged MLA bucket. Left whole, the SWA bucket would
|
||||
set the block width and the MLA group would fill under a third of
|
||||
every block; capped, it splits into groups no wider than the MLA
|
||||
page."""
|
||||
page.
|
||||
"""
|
||||
config = _shared_layout_config()
|
||||
specs: dict[str, KVCacheSpec] = {}
|
||||
for layer, ratio in ((2, 2), (8, 2), (14, 2), (20, 1)):
|
||||
@@ -606,7 +611,8 @@ class TestCompressorRingGroup:
|
||||
"""DeepSeek-V4.1 compressor state: a CircularBufferSpec ring (capacity
|
||||
8) beside 128-token paged MLA groups and 64-token SWA groups. The ring
|
||||
claims one block per request, never hashes, and does not disturb the
|
||||
scheduler/hash block sizes the paged groups imply."""
|
||||
scheduler/hash block sizes the paged groups imply.
|
||||
"""
|
||||
config = _shared_layout_config()
|
||||
config.cache_config.enable_prefix_caching = True
|
||||
specs: dict[str, KVCacheSpec] = {
|
||||
|
||||
@@ -367,7 +367,8 @@ def test_free_request_with_duplicate_mm_hashes():
|
||||
"""Freeing a request whose two inputs share the same mm_hash must fully
|
||||
clean up request_cached_ids. After the first free_encoder_input call,
|
||||
cached[mm_hash] becomes empty; the second call must still remove the
|
||||
remaining input_id from request_cached_ids."""
|
||||
remaining input_id from request_cached_ids.
|
||||
"""
|
||||
manager = EncoderCacheManager(cache_size=20)
|
||||
|
||||
req = MockRequest("r1", ["imgA", "imgA"], [4, 4])
|
||||
@@ -383,7 +384,8 @@ def test_free_request_with_duplicate_mm_hashes():
|
||||
|
||||
def test_duplicate_mm_hash_stays_referenced_until_last_free():
|
||||
"""`cached` holds one reference per request, not per position, so freeing
|
||||
the first of two occurrences must not release the entry."""
|
||||
the first of two occurrences must not release the entry.
|
||||
"""
|
||||
manager = EncoderCacheManager(cache_size=20)
|
||||
req = MockRequest("r1", ["imgA", "imgA"], [4, 4])
|
||||
|
||||
@@ -404,7 +406,8 @@ def test_duplicate_mm_hash_stays_referenced_until_last_free():
|
||||
def test_duplicate_mm_hash_is_not_evicted_before_last_use():
|
||||
"""An item repeated within one request (an image carried across
|
||||
conversation turns) must not become reclaimable capacity while a later
|
||||
occurrence still has to be spliced in, or the encoder recomputes it."""
|
||||
occurrence still has to be spliced in, or the encoder recomputes it.
|
||||
"""
|
||||
manager = EncoderCacheManager(cache_size=8)
|
||||
req = MockRequest("r1", ["imgA", "imgA"], [4, 4])
|
||||
other = MockRequest("r2", ["imgB"], [8])
|
||||
|
||||
@@ -1069,9 +1069,7 @@ def _stats(requests: int, queries: int, hits: int) -> PrefixCacheStats:
|
||||
|
||||
|
||||
def test_metrics():
|
||||
"""
|
||||
Test the prefix caching metrics.
|
||||
"""
|
||||
"""Test the prefix caching metrics."""
|
||||
metrics = CachingMetrics(max_recent_requests=5)
|
||||
assert metrics.hit_rate == 0.0
|
||||
|
||||
@@ -1101,9 +1099,7 @@ def test_metrics():
|
||||
|
||||
|
||||
def test_metrics_empty_stats():
|
||||
"""
|
||||
Test the prefix caching metrics with empty stats.
|
||||
"""
|
||||
"""Test the prefix caching metrics with empty stats."""
|
||||
metrics = CachingMetrics(max_recent_requests=5)
|
||||
metrics.observe(_stats(0, 0, 0))
|
||||
metrics.observe(_stats(1, 20, 9))
|
||||
@@ -1903,7 +1899,7 @@ def test_get_max_concurrency_for_kv_cache_config():
|
||||
|
||||
|
||||
def test_allocate_with_lookahead():
|
||||
"""Verify that lookahead tokens correctly affect block allocation"""
|
||||
"""Verify that lookahead tokens correctly affect block allocation."""
|
||||
block_size = 4
|
||||
config = KVCacheConfig(
|
||||
num_blocks=10,
|
||||
@@ -2597,7 +2593,8 @@ def test_get_kv_cache_config_kpool_tail_coowns_indexer_tensor():
|
||||
"""The kpool tail parasitizes the indexer tensors instead of getting its
|
||||
own: sibling idx/tail tensors paired by layer order, zero standalone tail
|
||||
bytes, one shared block per request, and no prefix-caching leakage from
|
||||
the kpool-sized scratch group."""
|
||||
the kpool-sized scratch group.
|
||||
"""
|
||||
model_config = ModelConfig(max_model_len=8192)
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
|
||||
@@ -2668,7 +2665,8 @@ def test_get_kv_cache_config_kpool_tail_coowns_indexer_tensor():
|
||||
|
||||
def test_glm5_kpool_tail_does_not_drag_hash_block_size():
|
||||
"""The tail's kpool-sized scratch block (4 tokens) must not constrain the
|
||||
prefix-cache hash granularity: participating groups alone decide it."""
|
||||
prefix-cache hash granularity: participating groups alone decide it.
|
||||
"""
|
||||
model_config = ModelConfig(max_model_len=8192)
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
|
||||
@@ -2744,7 +2742,8 @@ def test_get_kv_cache_config_mamba_hybrid_sharing_infeasible_no_indexer():
|
||||
|
||||
def test_get_kv_cache_config_mamba_hybrid_sharing_prepadded_mamba():
|
||||
"""Platform-prepadded mamba pages (mamba_page_size_padded hint) must not
|
||||
disable slot sharing; the layout re-pads them to the MLA page."""
|
||||
disable slot sharing; the layout re-pads them to the MLA page.
|
||||
"""
|
||||
model_config = ModelConfig(max_model_len=8192)
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
|
||||
@@ -2775,7 +2774,8 @@ def test_get_kv_cache_config_mamba_hybrid_sharing_prepadded_mamba():
|
||||
def test_get_kv_cache_config_mamba_hybrid_sharing_pp_balanced_projection():
|
||||
"""Round-robin mamba grouping keeps practical PP splits balanced: every
|
||||
stage's largest projected mamba group slice fits its projected MLA
|
||||
layers, so per-stage slot tensors all have an MLA owner."""
|
||||
layers, so per-stage slot tensors all have an MLA owner.
|
||||
"""
|
||||
model_config = ModelConfig(max_model_len=8192)
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
|
||||
@@ -2819,7 +2819,8 @@ def test_get_kv_cache_config_mamba_hybrid_sharing_pp_group_count_bump(monkeypatc
|
||||
"""PP=4's default partition [11,11,12,11] leaves stage 0 with 2 MLA
|
||||
layers but a round-robin slice of 3 under the minimum 4 mamba groups;
|
||||
grouping bumps to 5 groups so every stage's projection keeps sharing
|
||||
on instead of silently falling back on stage 0."""
|
||||
on instead of silently falling back on stage 0.
|
||||
"""
|
||||
from vllm.config import ParallelConfig
|
||||
from vllm.distributed.utils import get_pp_indices
|
||||
|
||||
@@ -2868,7 +2869,8 @@ def test_get_kv_cache_config_mamba_hybrid_sharing_pp_starved_stage(monkeypatch):
|
||||
def test_get_kv_cache_config_mamba_hybrid_sharing_beats_cross_layers_flag():
|
||||
"""Hybrid slot sharing must take precedence over the experimental
|
||||
enable_cross_layers_blocks packed layout: generic packing would give the
|
||||
MLA slots a strided view, breaking the contiguous virtual split."""
|
||||
MLA slots a strided view, breaking the contiguous virtual split.
|
||||
"""
|
||||
model_config = ModelConfig(max_model_len=8192)
|
||||
kv_transfer_config = KVTransferConfig(
|
||||
kv_connector="NixlConnector",
|
||||
@@ -3170,7 +3172,8 @@ def test_multi_run_layer_compact_strides_place_hoisted_heads():
|
||||
"""A layer-compact run region is its own dense allocation: under LHBNC the head
|
||||
groups sit between the layers and the blocks, so a run's block stride is one head
|
||||
group's slice, not the whole page (regression test for setStorage out-of-bounds
|
||||
on ROCm hybrid models)."""
|
||||
on ROCm hybrid models).
|
||||
"""
|
||||
full = new_kv_cache_spec()
|
||||
swa = new_sliding_window_spec(sliding_window=full.block_size * 2)
|
||||
page = full.page_size_bytes
|
||||
@@ -3249,7 +3252,8 @@ def test_mla_with_incompatible_swa_uses_one_full_allocation_group(caplog_vllm):
|
||||
def test_hidden_states_with_tp_scales_page_size():
|
||||
"""When TP shrinks KV pages below the hidden-state per-token cost,
|
||||
get_kv_cache_groups must scale up target block sizes so that the
|
||||
common page accommodates the unsharded hidden states."""
|
||||
common page accommodates the unsharded hidden states.
|
||||
"""
|
||||
# Simulate TP=4 sharding a model with 8 KV heads → 2 per rank.
|
||||
# KV page = block_size(16) * num_kv_heads(2) * head_size(64) * dtype(2)
|
||||
# = 16 * 2 * 64 * 2 = 4096 bytes.
|
||||
@@ -3922,8 +3926,7 @@ def test_unify_kv_cache_spec_page_size_mamba():
|
||||
|
||||
|
||||
def test_hma_not_disabled_when_kv_events_enabled():
|
||||
"""
|
||||
Test enabling KV events must not force disable_hybrid_kv_cache_manager to True.
|
||||
"""Test enabling KV events must not force disable_hybrid_kv_cache_manager to True.
|
||||
|
||||
This test guards against that regression by verifying that a VllmConfig
|
||||
with kv_events_config set still resolves disable_hybrid_kv_cache_manager
|
||||
@@ -4080,7 +4083,8 @@ def test_check_enough_kv_cache_memory_reserves_null_block():
|
||||
|
||||
def test_is_full_attention_spec_unwraps_uniform_type_specs():
|
||||
"""``UniformTypeKVCacheSpecs`` is not a ``FullAttentionSpec``, so callers
|
||||
scanning groups with a bare isinstance miss DeepSeek-V4-shaped configs."""
|
||||
scanning groups with a bare isinstance miss DeepSeek-V4-shaped configs.
|
||||
"""
|
||||
common = dict(num_kv_heads=1, head_size=1, dtype=torch.float32)
|
||||
full = FullAttentionSpec(block_size=4, **common)
|
||||
mla = MLAAttentionSpec(block_size=4, **common)
|
||||
@@ -4249,7 +4253,8 @@ def test_no_warning_when_draft_group_is_identified(caplog_vllm):
|
||||
|
||||
def _deepseek_v4_specs(model_version="deepseek_v4"):
|
||||
"""DeepseekV4-shaped specs: full MLA layers plus sliding-window MLA layers
|
||||
at differing page sizes, with the MTP draft layer registered last."""
|
||||
at differing page sizes, with the MTP draft layer registered last.
|
||||
"""
|
||||
return {
|
||||
"model.layers.0.self_attn.attn": new_mla_spec(),
|
||||
"model.layers.1.self_attn.attn": new_mla_spec(),
|
||||
|
||||
@@ -17,8 +17,7 @@ def new_kv_cache_spec():
|
||||
|
||||
|
||||
def test_initialize_kv_cache_for_kv_sharing_different_attn_groups():
|
||||
"""
|
||||
Test initializing KV cache sharing with different attention groups.
|
||||
"""Test initializing KV cache sharing with different attention groups.
|
||||
Layers in the same KV cache group might be placed in different attn groups
|
||||
if they have different attention backends.
|
||||
"""
|
||||
@@ -50,8 +49,7 @@ def test_initialize_kv_cache_for_kv_sharing_different_attn_groups():
|
||||
|
||||
|
||||
def test_initialize_kv_cache_for_kv_sharing_same_attn_groups():
|
||||
"""
|
||||
Test case assuming that all layers in the same KV cache group have the same
|
||||
"""Test case assuming that all layers in the same KV cache group have the same
|
||||
attention backends. This is true for most models.
|
||||
"""
|
||||
shared_kv_cache_layers = {
|
||||
@@ -79,8 +77,7 @@ def test_initialize_kv_cache_for_kv_sharing_same_attn_groups():
|
||||
|
||||
|
||||
def test_initialize_kv_cache_for_kv_sharing_no_attn_groups():
|
||||
"""
|
||||
Test KV sharing set up when no attention groups are provided.
|
||||
"""Test KV sharing set up when no attention groups are provided.
|
||||
This is the case for the TPU model runner, which doesn't have
|
||||
support for attention groups yet.
|
||||
"""
|
||||
|
||||
@@ -55,7 +55,8 @@ def _mm_feature(offset: int, length: int) -> MultiModalFeatureSpec:
|
||||
|
||||
def test_strip_covered_mm_data() -> None:
|
||||
"""Items fully inside the computed prefix lose their data; items touching
|
||||
the uncomputed region keep it; already-None data stays None."""
|
||||
the uncomputed region keep it; already-None data stays None.
|
||||
"""
|
||||
from dataclasses import replace
|
||||
|
||||
covered = _mm_feature(offset=0, length=100)
|
||||
@@ -107,7 +108,8 @@ def _mm_feature_mixed(offset: int, length: int) -> MultiModalFeatureSpec:
|
||||
|
||||
def test_strip_covered_mm_data_mrope() -> None:
|
||||
"""For M-RoPE models, covered items keep their keep_on_cpu metadata fields
|
||||
(the worker needs them to compute positions); payload fields are dropped."""
|
||||
(the worker needs them to compute positions); payload fields are dropped.
|
||||
"""
|
||||
covered = _mm_feature_mixed(offset=0, length=100)
|
||||
uncovered = _mm_feature_mixed(offset=300, length=100)
|
||||
|
||||
@@ -127,7 +129,8 @@ def test_strip_covered_mm_data_shm_address_item() -> None:
|
||||
"""SHM address descriptors must survive stripping even when covered: the
|
||||
worker needs the address to resolve the payload and to acknowledge the
|
||||
sender's writer reference. Stripping it crashed the SHM receiver cache on
|
||||
the second identical request (vllm-project/vllm#54994)."""
|
||||
the second identical request (vllm-project/vllm#54994).
|
||||
"""
|
||||
address_item = MultiModalKwargsItem(
|
||||
{
|
||||
"address": MultiModalFieldElem(data=4096, field=MultiModalBatchedField()),
|
||||
|
||||
@@ -126,7 +126,8 @@ def make_kv_cache_manager(kv_cache_config: KVCacheConfig, **kwargs) -> KVCacheMa
|
||||
"""Build a ``KVCacheManager``, deriving ``scheduler_block_size`` from the
|
||||
config (LCM of group block sizes) unless explicitly provided. This mirrors
|
||||
``resolve_kv_cache_block_sizes`` for the non-context-parallel case used by
|
||||
these tests, so callers don't have to pass it at every site."""
|
||||
these tests, so callers don't have to pass it at every site.
|
||||
"""
|
||||
kwargs.setdefault(
|
||||
"scheduler_block_size",
|
||||
lcm(*(g.kv_cache_spec.block_size for g in kv_cache_config.kv_cache_groups)),
|
||||
@@ -1826,8 +1827,7 @@ def _test_partial_request_hit(
|
||||
def _make_hybrid_kv_cache_config(
|
||||
block_size: int, num_blocks: int, spec_types: list[str]
|
||||
) -> KVCacheConfig:
|
||||
"""
|
||||
Create a KVCacheConfig with the specified spec types.
|
||||
"""Create a KVCacheConfig with the specified spec types.
|
||||
|
||||
Args:
|
||||
block_size: The block size for KV cache.
|
||||
@@ -1837,6 +1837,7 @@ def _make_hybrid_kv_cache_config(
|
||||
- "sliding_window": SlidingWindowSpec with window=2*block_size
|
||||
- "sliding_window_large": SlidingWindowSpec with window=4*block_size
|
||||
- "mamba": MambaSpec
|
||||
|
||||
"""
|
||||
spec_map = {
|
||||
"full": lambda: FullAttentionSpec(
|
||||
@@ -1938,8 +1939,7 @@ _HYBRID_MODEL_TEST_CASES = [
|
||||
|
||||
@pytest.mark.parametrize("spec_types", _HYBRID_MODEL_TEST_CASES)
|
||||
def test_prefill_hybrid_model_combinations(spec_types: list[str]):
|
||||
"""
|
||||
Test prefix caching with hybrid models containing various combinations of
|
||||
"""Test prefix caching with hybrid models containing various combinations of
|
||||
KV cache spec types.
|
||||
|
||||
This unified test covers:
|
||||
@@ -2021,8 +2021,7 @@ _EAGLE_HYBRID_MODEL_TEST_CASES = [
|
||||
def test_prefill_hybrid_model_combinations_eagle(
|
||||
spec_types: list[str], expect_hit_length: int
|
||||
):
|
||||
"""
|
||||
Test prefix caching with hybrid models (1 full attn + 1 other) with EAGLE.
|
||||
"""Test prefix caching with hybrid models (1 full attn + 1 other) with EAGLE.
|
||||
More complex hybrid models with EAGLE are not yet supported (see issue #32802).
|
||||
"""
|
||||
block_size = 16
|
||||
@@ -2489,8 +2488,7 @@ def test_evict():
|
||||
|
||||
|
||||
def test_hash_block_correct_reuse():
|
||||
"""
|
||||
This tests when a previously cached block is reused as a new block,
|
||||
"""This tests when a previously cached block is reused as a new block,
|
||||
its hash metadata should be correctly reset.
|
||||
"""
|
||||
block_size = 16
|
||||
@@ -2530,8 +2528,7 @@ def test_hash_block_correct_reuse():
|
||||
|
||||
|
||||
def test_computed_blocks_not_evicted():
|
||||
"""
|
||||
Test that the computed blocks are not evicted when getting new blocks
|
||||
"""Test that the computed blocks are not evicted when getting new blocks
|
||||
for a request if there are any other free blocks.
|
||||
"""
|
||||
block_size = 16
|
||||
@@ -2590,9 +2587,7 @@ def test_computed_blocks_not_evicted():
|
||||
|
||||
|
||||
def test_basic_prefix_caching_disabled():
|
||||
"""
|
||||
This tests that the prefix caching is disabled.
|
||||
"""
|
||||
"""This tests that the prefix caching is disabled."""
|
||||
block_size = 4
|
||||
manager = make_kv_cache_manager(
|
||||
make_kv_cache_config(block_size, 5),
|
||||
@@ -2639,11 +2634,9 @@ def test_basic_prefix_caching_disabled():
|
||||
|
||||
@pytest.mark.parametrize("hash_fn", [sha256, sha256_cbor])
|
||||
def test_cache_blocks(hash_fn):
|
||||
"""
|
||||
This is a unit test that tests the correctness of the _cache_full_blocks
|
||||
"""This is a unit test that tests the correctness of the _cache_full_blocks
|
||||
function of KVCacheManager.
|
||||
"""
|
||||
|
||||
block_size = 4
|
||||
block_pool = BlockPool(
|
||||
num_gpu_blocks=5,
|
||||
@@ -2688,9 +2681,7 @@ def test_cache_blocks(hash_fn):
|
||||
|
||||
|
||||
def test_cache_blocks_multi_group():
|
||||
"""
|
||||
This tests that blocks are cached correctly for different kv cache groups.
|
||||
"""
|
||||
"""This tests that blocks are cached correctly for different kv cache groups."""
|
||||
block_size = 4
|
||||
block_pool = BlockPool(
|
||||
num_gpu_blocks=10, enable_caching=True, hash_block_size=block_size
|
||||
@@ -2773,10 +2764,7 @@ def test_cache_blocks_multi_group():
|
||||
|
||||
|
||||
def test_mm_prefix_caching():
|
||||
"""
|
||||
This tests that the multi-modal prefix caching is correct.
|
||||
"""
|
||||
|
||||
"""This tests that the multi-modal prefix caching is correct."""
|
||||
block_size = 16
|
||||
manager = make_kv_cache_manager(
|
||||
make_kv_cache_config(block_size, 11),
|
||||
@@ -2881,8 +2869,7 @@ def test_mm_prefix_caching():
|
||||
|
||||
|
||||
def test_cache_key_salting():
|
||||
"""
|
||||
This tests that cache salts are applied during hashing and the cache
|
||||
"""This tests that cache salts are applied during hashing and the cache
|
||||
is separated cache as expected.
|
||||
"""
|
||||
block_size = 16
|
||||
@@ -2961,8 +2948,7 @@ def test_cache_key_salting():
|
||||
|
||||
|
||||
def test_prefill_not_enough_free_blocks_with_computed_blocks():
|
||||
"""
|
||||
This is a unit test that tests the correctness of the allocate_slots
|
||||
"""This is a unit test that tests the correctness of the allocate_slots
|
||||
when there is not enough free blocks. Specifically, when a request
|
||||
has computed blocks but cannot be allocated due to not enough free blocks,
|
||||
the computed blocks should not be touched.
|
||||
@@ -3344,7 +3330,8 @@ def test_kv_cache_events_with_lora(blocks_to_cache: int):
|
||||
@pytest.mark.parametrize("group_id", [0, 1, 2])
|
||||
def test_block_stored_event_group_idx(group_id: int):
|
||||
"""Test BlockStored events emitted by cache_full_blocks carry the correct
|
||||
group_idx."""
|
||||
group_idx.
|
||||
"""
|
||||
block_size = 4
|
||||
num_tokens = block_size * 2
|
||||
|
||||
@@ -3462,8 +3449,7 @@ def test_session_id_does_not_affect_prefix_cache_identity():
|
||||
|
||||
|
||||
def test_block_stored_event_group_idx_multiple_groups():
|
||||
"""
|
||||
Test BlockStored events for separate HMA groups that each carry the
|
||||
"""Test BlockStored events for separate HMA groups that each carry the
|
||||
correct group_idx.
|
||||
|
||||
Simulates the HMA scenario where full-attention blocks (group 0) and
|
||||
@@ -3585,8 +3571,7 @@ def test_block_stored_event_group_idx_out_of_bounds(monkeypatch):
|
||||
|
||||
@pytest.mark.parametrize("group_id", [0, 1, 2])
|
||||
def test_block_removed_event_group_idx(group_id: int):
|
||||
"""
|
||||
Test BlockRemoved events emitted on eviction carry the group_idx extracted
|
||||
"""Test BlockRemoved events emitted on eviction carry the group_idx extracted
|
||||
from the evicted block's BlockHashWithGroupId via get_group_id().
|
||||
"""
|
||||
block_size = 4
|
||||
@@ -3639,7 +3624,8 @@ def test_block_removed_event_group_idx(group_id: int):
|
||||
def test_emit_cached_block_events():
|
||||
"""emit_cached_block_events emits one BlockStored for already-cached
|
||||
(reused) prefix blocks, carrying the correct group_idx /
|
||||
parent_block_hash / token_ids, and without mutating block state."""
|
||||
parent_block_hash / token_ids, and without mutating block state.
|
||||
"""
|
||||
block_size = 4
|
||||
num_cached_blocks = 3
|
||||
kv_cache_group_id = 1
|
||||
@@ -3752,7 +3738,8 @@ def test_emit_cached_block_events_zero_cached():
|
||||
|
||||
def test_eagle_enabled_removes_last_block():
|
||||
"""Verify Eagle does NOT remove blocks when request
|
||||
length is divisible by block size."""
|
||||
length is divisible by block size.
|
||||
"""
|
||||
block_size = 16
|
||||
manager = make_kv_cache_manager(
|
||||
make_kv_cache_config(block_size, num_blocks=10),
|
||||
@@ -4155,7 +4142,8 @@ def test_hybrid_cache_blocks_swa_tail_window_only():
|
||||
(its right-to-left scan stops once a contiguous match is found). Blocks
|
||||
earlier in the segment can never serve a hit, so
|
||||
``HybridKVCacheCoordinator.cache_blocks`` should skip them rather than
|
||||
polluting the prefix-cache hash map."""
|
||||
polluting the prefix-cache hash map.
|
||||
"""
|
||||
block_size = 8
|
||||
# Full attn block_size=32, SWA block_size=8, sw=8 -> lcm=32.
|
||||
# tail = ceil(7/8) = 1; per_segment = 32/8 = 4.
|
||||
@@ -4226,7 +4214,8 @@ def test_hybrid_cache_blocks_clamped_to_lcm():
|
||||
Chunks past the last lcm-aligned boundary can never participate in a
|
||||
cache hit (find_longest_cache_hit always returns lcm-aligned hits), so
|
||||
caching them only pollutes the prefix-cache hash map and keeps blocks
|
||||
on the LRU list that could otherwise return to the free pool."""
|
||||
on the LRU list that could otherwise return to the free pool.
|
||||
"""
|
||||
block_size = 16
|
||||
# Full attn block_size=32, SWA block_size=16 -> lcm=32.
|
||||
kv_cache_config = KVCacheConfig(
|
||||
@@ -4361,7 +4350,8 @@ def test_hybrid_local_kv_retention_interval_aligns_in_manager():
|
||||
)
|
||||
def test_hybrid_local_kv_retention_interval_rejects_invalid(interval, expected_match):
|
||||
"""A retention interval that is negative or not a multiple of
|
||||
scheduler_block_size errors out at construction time."""
|
||||
scheduler_block_size errors out at construction time.
|
||||
"""
|
||||
block_size = 8
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=100,
|
||||
@@ -4831,7 +4821,8 @@ def test_block_lookup_cache_multi_blocks_per_key():
|
||||
def test_can_fit_full_sequence_swa_cap_admits_long_prompt():
|
||||
"""Hybrid full+SWA model with a pool sized at the startup minimum should
|
||||
admit a prompt longer than the SWA cap, because SlidingWindowManager
|
||||
recycles blocks during chunked prefill (issue #39734)."""
|
||||
recycles blocks during chunked prefill (issue #39734).
|
||||
"""
|
||||
block_size = 16
|
||||
sliding_window = 4 * block_size # 64 tokens
|
||||
max_num_batched_tokens = 8 * block_size # 128 tokens
|
||||
@@ -4926,7 +4917,8 @@ def test_full_sequence_admission_reserves_device_blocks_for_immediate_chunk(
|
||||
|
||||
def test_can_fit_full_sequence_full_attention_still_gates_oversized():
|
||||
"""The cap only loosens the SWA group; a prompt that exceeds the
|
||||
full-attention pool capacity must still be rejected."""
|
||||
full-attention pool capacity must still be rejected.
|
||||
"""
|
||||
block_size = 16
|
||||
sliding_window = 4 * block_size
|
||||
max_num_batched_tokens = 8 * block_size
|
||||
@@ -5045,7 +5037,8 @@ def _take_free_blocks(manager: KVCacheManager, num_blocks: int) -> list[KVCacheB
|
||||
without removing them. These ref_cnt==0 blocks stand in for evictable
|
||||
cache-hit blocks left behind by a previous (e.g. preempted) request, and
|
||||
sitting at the head guarantees a later group's external ``get_new_blocks``
|
||||
would contend for them on unpatched code (issue #33775)."""
|
||||
would contend for them on unpatched code (issue #33775).
|
||||
"""
|
||||
blocks: list[KVCacheBlock] = []
|
||||
head = manager.block_pool.free_block_queue.fake_free_list_head
|
||||
for _ in range(num_blocks):
|
||||
@@ -5056,7 +5049,8 @@ def _take_free_blocks(manager: KVCacheManager, num_blocks: int) -> list[KVCacheB
|
||||
|
||||
def _assert_no_double_allocation(manager: KVCacheManager, req_id: str) -> None:
|
||||
"""No physical block may be handed out twice across groups, and every
|
||||
block referenced by the request must have a live ref_cnt."""
|
||||
block referenced by the request must have a live ref_cnt.
|
||||
"""
|
||||
block_ids = manager.get_blocks(req_id).get_block_ids()
|
||||
flat = [block_id for group in block_ids for block_id in group]
|
||||
assert len(set(flat)) == len(flat), "Block IDs are not unique across groups"
|
||||
@@ -5083,7 +5077,8 @@ def _cross_group_cache_hit(
|
||||
) -> Request:
|
||||
"""Allocate ``req_id`` with a per-group local prefix hit plus external
|
||||
(connector) computed tokens, driving the coordinator's two-phase path.
|
||||
Returns the allocated request so callers can free it (e.g. to preempt)."""
|
||||
Returns the allocated request so callers can free it (e.g. to preempt).
|
||||
"""
|
||||
block_size = _two_phase_block_size(manager)
|
||||
hit_blocks = _take_free_blocks(manager, num_groups * local_blocks_per_group)
|
||||
cache_hit = KVCacheBlocks(
|
||||
@@ -5178,7 +5173,8 @@ def test_swa_free_split_keeps_cached_tail_ahead_of_scratch():
|
||||
"""Dense retention: freeing an SWA request must place its
|
||||
uncached scratch blocks at the front of the free queue (recycled first)
|
||||
and keep its cached checkpoint blocks at the back (retained for prefix
|
||||
hits). This split is always-on, independent of the retention interval."""
|
||||
hits). This split is always-on, independent of the retention interval.
|
||||
"""
|
||||
block_size = 8
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=100,
|
||||
@@ -5289,7 +5285,8 @@ def _make_pure_swa_manager(block_size, sliding_window, num_blocks=100, **kwargs)
|
||||
def test_pure_swa_retention_interval_caches_sparse_tails():
|
||||
"""Sparse retention must work for a pure-SWA single-group model, not just
|
||||
hybrid models: only the per-interval tails plus the latest replay tail are
|
||||
cached, and a replay still hits the latest replayable boundary."""
|
||||
cached, and a replay still hits the latest replayable boundary.
|
||||
"""
|
||||
block_size = 16
|
||||
manager = _make_pure_swa_manager(
|
||||
block_size, sliding_window=block_size, retention_interval=64
|
||||
@@ -5361,7 +5358,8 @@ def test_pure_swa_retention_latest_only():
|
||||
|
||||
def test_pure_swa_dense_retention_caches_all():
|
||||
"""With retention set to ``None``, a pure-SWA model keeps dense behavior:
|
||||
every block boundary is a potential hit, so all blocks are cached."""
|
||||
every block boundary is a potential hit, so all blocks are cached.
|
||||
"""
|
||||
block_size = 16
|
||||
manager = _make_pure_swa_manager(block_size, sliding_window=block_size)
|
||||
|
||||
@@ -5390,7 +5388,8 @@ def test_mamba_reachable_block_mask_sparsifies_retention():
|
||||
"""Mamba state-snapshot retention: with a configured retention interval,
|
||||
the manager keeps one cached state per interval-sized segment (plus the
|
||||
latest replay boundary) instead of a snapshot per block, which is what
|
||||
lets a small attention block_size avoid Mamba dominating the KV pool."""
|
||||
lets a small attention block_size avoid Mamba dominating the KV pool.
|
||||
"""
|
||||
from vllm.v1.core.single_type_kv_cache_manager import MambaManager
|
||||
|
||||
block_size = 16
|
||||
@@ -5428,7 +5427,8 @@ def test_mamba_reachable_block_mask_pins_shared_prefix():
|
||||
"""A Marconi-detected shared prefix (``shared_prefix_boundary``) lands before
|
||||
``num_prompt`` so the replay-boundary rule alone would drop it. The mask must
|
||||
pin the single state block ending on that boundary so sparse retention does
|
||||
not defeat cross-request shared-prefix reuse."""
|
||||
not defeat cross-request shared-prefix reuse.
|
||||
"""
|
||||
from vllm.v1.core.single_type_kv_cache_manager import MambaManager
|
||||
|
||||
block_size = 16
|
||||
@@ -5476,7 +5476,8 @@ def test_mamba_shared_prefix_survives_zero_retention():
|
||||
detection) keeps its Mamba state block cached under
|
||||
``prefix_cache_retention_interval=0``, which otherwise retains only the
|
||||
end-of-prompt replay boundary. Without this, a shared prefix (junction
|
||||
before ``num_prompt``) would be recomputed by every sharing request."""
|
||||
before ``num_prompt``) would be recomputed by every sharing request.
|
||||
"""
|
||||
block_size = 16
|
||||
|
||||
# 16-block (256-token) prompt; replay boundary is block 240 // 16 - 1 = 14.
|
||||
@@ -5518,7 +5519,8 @@ def test_mamba_shared_prefix_reuse_under_zero_retention():
|
||||
detecting request must stay reusable by a later request under
|
||||
``prefix_cache_retention_interval=0``. Without the pin the junction is
|
||||
masked out and the later request misses; with it (and under dense) the reuse
|
||||
is preserved."""
|
||||
is preserved.
|
||||
"""
|
||||
block_size = 16
|
||||
|
||||
def last_req_hit(retention, pin):
|
||||
@@ -5601,7 +5603,8 @@ def test_swa_reachable_block_mask_final_partial_segment():
|
||||
def test_swa_reachable_block_mask_pins_shared_prefix():
|
||||
"""SWA analog of the Mamba pin: the shared-prefix junction must keep the
|
||||
``need``-block sliding-window tail ending on that boundary (not a single
|
||||
block), so a windowed hit can land there under sparse retention."""
|
||||
block), so a windowed hit can land there under sparse retention.
|
||||
"""
|
||||
from vllm.v1.core.single_type_kv_cache_manager import SlidingWindowManager
|
||||
|
||||
block_size = 16
|
||||
@@ -5646,7 +5649,8 @@ def test_swa_reachable_block_mask_pins_shared_prefix():
|
||||
def test_swa_reachable_block_mask_with_dcp_scaling():
|
||||
"""DCP shards each block's KV across ranks, scaling the effective block size.
|
||||
Verify that dcp_world_size > 1 scales block size in reachability calculations,
|
||||
producing different masks than dcp_world_size=1."""
|
||||
producing different masks than dcp_world_size=1.
|
||||
"""
|
||||
from vllm.v1.core.single_type_kv_cache_manager import SlidingWindowManager
|
||||
|
||||
block_size = 16
|
||||
@@ -5705,7 +5709,8 @@ def test_swa_reachable_block_mask_sub_block_alignment_is_dense(
|
||||
multiple of the DCP-scaled block size) cannot be represented exactly. This
|
||||
happens for hybrid offloading (e.g. Gemma), where alignment_tokens is the
|
||||
full-attention chunk size and need not divide the SWA block size. The mask
|
||||
must fall back to dense (None) rather than raise."""
|
||||
must fall back to dense (None) rather than raise.
|
||||
"""
|
||||
from vllm.v1.core.single_type_kv_cache_manager import SlidingWindowManager
|
||||
|
||||
spec = SlidingWindowSpec(
|
||||
@@ -5733,7 +5738,8 @@ def test_mamba_reachable_block_mask_ignores_dcp():
|
||||
"""Mamba uses TP, not DCP: each rank holds the full recurrent state, so a
|
||||
state block spans kv_cache_spec.block_size tokens regardless of DCP. The
|
||||
mask must not scale by dcp_world_size, so retention granularity is
|
||||
identical for any DCP world size."""
|
||||
identical for any DCP world size.
|
||||
"""
|
||||
from vllm.v1.core.single_type_kv_cache_manager import MambaManager
|
||||
|
||||
spec = MambaSpec(
|
||||
@@ -5767,7 +5773,8 @@ def test_mamba_reachable_block_mask_ignores_dcp():
|
||||
|
||||
def test_mamba_reachable_block_mask_large_dcp_stays_sparse():
|
||||
"""A large DCP world size must not scale the Mamba block size, so it can
|
||||
never collapse sparse retention into dense caching."""
|
||||
never collapse sparse retention into dense caching.
|
||||
"""
|
||||
from vllm.v1.core.single_type_kv_cache_manager import MambaManager
|
||||
|
||||
spec = MambaSpec(
|
||||
@@ -5802,7 +5809,8 @@ def test_swa_shared_prefix_reuse_under_zero_retention():
|
||||
"""SWA cross-request analog: a partial shared prefix's sliding-window tail
|
||||
must stay reusable under ``prefix_cache_retention_interval=0``. Without
|
||||
the pin the junction window is masked out and a later request misses; with
|
||||
it (and under dense) reuse is preserved."""
|
||||
it (and under dense) reuse is preserved.
|
||||
"""
|
||||
block_size = 16
|
||||
|
||||
def last_req_hit(retention, pin):
|
||||
|
||||
@@ -14,10 +14,10 @@ pytestmark = pytest.mark.cpu_test
|
||||
|
||||
|
||||
class TestCheckSequenceRepetition:
|
||||
"""Unit tests for the check_sequence_repetition function"""
|
||||
"""Unit tests for the check_sequence_repetition function."""
|
||||
|
||||
def test_simple_repetition_detected(self):
|
||||
"""Test detection of simple repetitive patterns"""
|
||||
"""Test detection of simple repetitive patterns."""
|
||||
token_ids = [1, 2, 3, 1, 2, 3, 1, 2, 3]
|
||||
params = RepetitionDetectionParams(
|
||||
max_pattern_size=3,
|
||||
@@ -27,7 +27,7 @@ class TestCheckSequenceRepetition:
|
||||
assert check_sequence_repetition(token_ids, params)
|
||||
|
||||
def test_repetition_below_min_count(self):
|
||||
"""Test that pattern below min_count is not detected"""
|
||||
"""Test that pattern below min_count is not detected."""
|
||||
token_ids = [1, 2, 3, 1, 2, 3]
|
||||
params = RepetitionDetectionParams(
|
||||
max_pattern_size=3,
|
||||
@@ -37,7 +37,7 @@ class TestCheckSequenceRepetition:
|
||||
assert not check_sequence_repetition(token_ids, params)
|
||||
|
||||
def test_two_token_pattern(self):
|
||||
"""Test detection of 2-token patterns"""
|
||||
"""Test detection of 2-token patterns."""
|
||||
token_ids = [1, 2, 1, 2, 1, 2, 1, 2]
|
||||
params = RepetitionDetectionParams(
|
||||
max_pattern_size=5,
|
||||
@@ -47,7 +47,7 @@ class TestCheckSequenceRepetition:
|
||||
assert check_sequence_repetition(token_ids, params)
|
||||
|
||||
def test_no_repetition_varied_sequence(self):
|
||||
"""Test that non-repetitive sequences are not flagged"""
|
||||
"""Test that non-repetitive sequences are not flagged."""
|
||||
token_ids = [1, 2, 3, 4, 5, 6, 7, 8, 9]
|
||||
params = RepetitionDetectionParams(
|
||||
max_pattern_size=5,
|
||||
@@ -57,7 +57,7 @@ class TestCheckSequenceRepetition:
|
||||
assert not check_sequence_repetition(token_ids, params)
|
||||
|
||||
def test_partial_repetition_not_detected(self):
|
||||
"""Test that incomplete repetitions are not detected"""
|
||||
"""Test that incomplete repetitions are not detected."""
|
||||
token_ids = [1, 2, 3, 1, 2, 3, 1, 2, 4]
|
||||
params = RepetitionDetectionParams(
|
||||
max_pattern_size=3,
|
||||
@@ -67,7 +67,7 @@ class TestCheckSequenceRepetition:
|
||||
assert not check_sequence_repetition(token_ids, params)
|
||||
|
||||
def test_empty_token_list(self):
|
||||
"""Test with empty token list"""
|
||||
"""Test with empty token list."""
|
||||
params = RepetitionDetectionParams(
|
||||
max_pattern_size=3,
|
||||
min_pattern_size=2,
|
||||
@@ -76,19 +76,19 @@ class TestCheckSequenceRepetition:
|
||||
assert not check_sequence_repetition([], params)
|
||||
|
||||
def test_detection_disabled_max_size_zero(self):
|
||||
"""Test that zero max_pattern_size disables detection"""
|
||||
"""Test that zero max_pattern_size disables detection."""
|
||||
token_ids = [1, 2, 1, 2, 1, 2]
|
||||
params = RepetitionDetectionParams()
|
||||
assert not check_sequence_repetition(token_ids, params)
|
||||
|
||||
def test_invalid_min_count(self):
|
||||
"""Test that min_count < 2 returns False"""
|
||||
"""Test that min_count < 2 returns False."""
|
||||
token_ids = [1, 2, 1, 2]
|
||||
params = RepetitionDetectionParams()
|
||||
assert not check_sequence_repetition(token_ids, params)
|
||||
|
||||
def test_repetition_at_end_of_sequence(self):
|
||||
"""Test detection when repetition occurs at the end"""
|
||||
"""Test detection when repetition occurs at the end."""
|
||||
token_ids = [1, 2, 3, 4, 5, 6, 5, 6, 5, 6]
|
||||
params = RepetitionDetectionParams(
|
||||
max_pattern_size=3,
|
||||
@@ -98,7 +98,7 @@ class TestCheckSequenceRepetition:
|
||||
assert check_sequence_repetition(token_ids, params)
|
||||
|
||||
def test_large_pattern_many_repetitions(self):
|
||||
"""Test large pattern repeated many times"""
|
||||
"""Test large pattern repeated many times."""
|
||||
token_ids = [1, 2, 3, 4, 5, 6, 7, 8] * 5
|
||||
params = RepetitionDetectionParams(
|
||||
max_pattern_size=10,
|
||||
@@ -114,10 +114,10 @@ class TestCheckSequenceRepetition:
|
||||
|
||||
|
||||
class TestRepetitionDetectionIntegration:
|
||||
"""Integration tests for repetition detection in check_stop"""
|
||||
"""Integration tests for repetition detection in check_stop."""
|
||||
|
||||
def test_basic_repetition_stops_generation(self):
|
||||
"""Test that repetition is detected and stops generation"""
|
||||
"""Test that repetition is detected and stops generation."""
|
||||
params = SamplingParams(
|
||||
max_tokens=100,
|
||||
repetition_detection=RepetitionDetectionParams(
|
||||
@@ -138,7 +138,7 @@ class TestRepetitionDetectionIntegration:
|
||||
assert request.stop_reason == "repetition_detected"
|
||||
|
||||
def test_detection_disabled_no_stop(self):
|
||||
"""Test that disabled detection doesn't stop generation"""
|
||||
"""Test that disabled detection doesn't stop generation."""
|
||||
params = SamplingParams(
|
||||
max_tokens=100,
|
||||
)
|
||||
@@ -152,7 +152,7 @@ class TestRepetitionDetectionIntegration:
|
||||
assert not check_stop(request, max_model_len=1024)
|
||||
|
||||
def test_repetition_respects_min_tokens(self):
|
||||
"""Test that repetition detection respects min_tokens"""
|
||||
"""Test that repetition detection respects min_tokens."""
|
||||
params = SamplingParams(
|
||||
min_tokens=10,
|
||||
max_tokens=100,
|
||||
@@ -172,7 +172,7 @@ class TestRepetitionDetectionIntegration:
|
||||
assert not check_stop(request, max_model_len=1024)
|
||||
|
||||
def test_no_repetition_continues_generation(self):
|
||||
"""Test that non-repetitive tokens don't stop generation"""
|
||||
"""Test that non-repetitive tokens don't stop generation."""
|
||||
params = SamplingParams(
|
||||
max_tokens=100,
|
||||
repetition_detection=RepetitionDetectionParams(
|
||||
@@ -191,7 +191,7 @@ class TestRepetitionDetectionIntegration:
|
||||
assert not check_stop(request, max_model_len=1024)
|
||||
|
||||
def test_pattern_at_size_boundary(self):
|
||||
"""Test detection at exact pattern size boundary"""
|
||||
"""Test detection at exact pattern size boundary."""
|
||||
params = SamplingParams(
|
||||
max_tokens=100,
|
||||
repetition_detection=RepetitionDetectionParams(
|
||||
@@ -211,7 +211,7 @@ class TestRepetitionDetectionIntegration:
|
||||
assert request.status == RequestStatus.FINISHED_REPETITION
|
||||
|
||||
def test_multiple_pattern_sizes_checked(self):
|
||||
"""Test that function checks pattern sizes in range"""
|
||||
"""Test that function checks pattern sizes in range."""
|
||||
params = SamplingParams(
|
||||
max_tokens=100,
|
||||
repetition_detection=RepetitionDetectionParams(
|
||||
@@ -231,7 +231,7 @@ class TestRepetitionDetectionIntegration:
|
||||
assert request.status == RequestStatus.FINISHED_REPETITION
|
||||
|
||||
def test_eos_takes_precedence_over_repetition(self):
|
||||
"""Test that EOS token stops before repetition check"""
|
||||
"""Test that EOS token stops before repetition check."""
|
||||
params = SamplingParams(
|
||||
max_tokens=100,
|
||||
stop_token_ids=[999],
|
||||
@@ -252,7 +252,7 @@ class TestRepetitionDetectionIntegration:
|
||||
assert request.status == RequestStatus.FINISHED_STOPPED
|
||||
|
||||
def test_min_pattern_size_filters_small_patterns(self):
|
||||
"""Test that min_pattern_size filters out smaller patterns"""
|
||||
"""Test that min_pattern_size filters out smaller patterns."""
|
||||
params = SamplingParams(
|
||||
max_tokens=100,
|
||||
repetition_detection=RepetitionDetectionParams(
|
||||
@@ -271,7 +271,7 @@ class TestRepetitionDetectionIntegration:
|
||||
assert not check_stop(request, max_model_len=1024)
|
||||
|
||||
def test_high_repetition_threshold(self):
|
||||
"""Test that high min_count requires many repetitions"""
|
||||
"""Test that high min_count requires many repetitions."""
|
||||
params = SamplingParams(
|
||||
max_tokens=100,
|
||||
repetition_detection=RepetitionDetectionParams(
|
||||
|
||||
@@ -295,7 +295,7 @@ def test_schedule_multimodal_requests():
|
||||
|
||||
|
||||
def test_async_scheduling_pp_allows_rescheduling_with_output_placeholders():
|
||||
"""Async scheduling + PP: allow multi-step in-flight scheduling per request"""
|
||||
"""Async scheduling + PP: allow multi-step in-flight scheduling per request."""
|
||||
scheduler = create_scheduler(async_scheduling=True, pipeline_parallel_size=2)
|
||||
(req,) = create_requests(num_requests=1, num_tokens=8)
|
||||
scheduler.add_request(req)
|
||||
@@ -538,7 +538,8 @@ def test_throttle_defers_inflight_prefill_chunk():
|
||||
"""DP prefill balancing throttles ALL prefill compute on a throttled step,
|
||||
not just new admissions: an in-progress (chunked) prefill already in the
|
||||
running queue is also deferred, so the step runs decode-only, while a
|
||||
separate decode keeps being scheduled."""
|
||||
separate decode keeps being scheduled.
|
||||
"""
|
||||
scheduler = create_scheduler(
|
||||
max_num_seqs=16, max_num_batched_tokens=50, enable_chunked_prefill=True
|
||||
)
|
||||
@@ -592,7 +593,8 @@ def test_throttle_capacity_bound_guard_admits():
|
||||
"""Saturation guard: if a cadence-aligned release step cannot drain the
|
||||
waiting prefill queue (it ran out of token budget), the throttle backs off on
|
||||
the next step so the backlog cannot grow into a TTFT avalanche -- prefills are
|
||||
admitted even though throttle_prefills is set."""
|
||||
admitted even though throttle_prefills is set.
|
||||
"""
|
||||
scheduler = create_scheduler(
|
||||
max_num_seqs=16, max_num_batched_tokens=200, enable_chunked_prefill=True
|
||||
)
|
||||
@@ -785,7 +787,7 @@ def test_update_from_output_routes_sampling_masks_by_request():
|
||||
|
||||
|
||||
def test_stop_via_update_from_output():
|
||||
"""Test stopping behavior through update_from_output"""
|
||||
"""Test stopping behavior through update_from_output."""
|
||||
scheduler = create_scheduler(num_speculative_tokens=1)
|
||||
|
||||
# Test case 1: Stop on EOS token
|
||||
@@ -1291,7 +1293,8 @@ def test_pending_hisparse_reclamation_defers_preemption(monkeypatch):
|
||||
|
||||
def test_prefix_cache_query_not_inflated_by_connector_defer():
|
||||
"""The GPU prefix-cache query is recorded at admission, so a request the
|
||||
connector defers several times is counted once, not once per retry."""
|
||||
connector defers several times is counted once, not once per retry.
|
||||
"""
|
||||
num_defers_before_matching = 3
|
||||
scheduler = create_scheduler(
|
||||
enable_prefix_caching=True,
|
||||
@@ -1319,7 +1322,8 @@ def test_prefix_cache_query_not_inflated_by_connector_defer():
|
||||
|
||||
def test_preemption_re_records_prefix_cache_query():
|
||||
"""A preempted request re-enters the lookup on resume, so its recomputation
|
||||
is counted again into the preempted stats."""
|
||||
is counted again into the preempted stats.
|
||||
"""
|
||||
scheduler = create_scheduler(enable_prefix_caching=True)
|
||||
request = create_requests(num_requests=1)[0]
|
||||
scheduler.add_request(request)
|
||||
@@ -1355,7 +1359,8 @@ def test_preemption_re_records_prefix_cache_query():
|
||||
|
||||
def test_prefix_cache_stats_not_recorded_when_caching_disabled():
|
||||
"""With prefix caching off there is no local lookup, so admitting a request
|
||||
records no phantom miss."""
|
||||
records no phantom miss.
|
||||
"""
|
||||
scheduler = create_scheduler(enable_prefix_caching=False)
|
||||
for request in create_requests(num_requests=2):
|
||||
scheduler.add_request(request)
|
||||
@@ -1369,7 +1374,8 @@ def test_prefix_cache_stats_not_recorded_when_caching_disabled():
|
||||
|
||||
def test_prefix_cache_stats_counted_once_for_retried_then_scheduled_request():
|
||||
"""A real cache hit rejected once by allocate_slots and admitted on the next
|
||||
step is counted exactly once, hits included."""
|
||||
step is counted exactly once, hits included.
|
||||
"""
|
||||
block_size = 16
|
||||
scheduler = create_scheduler(
|
||||
enable_prefix_caching=True,
|
||||
@@ -2130,7 +2136,6 @@ def _assert_right_scheduler_output(
|
||||
expected_num_scheduled_tokens: int,
|
||||
):
|
||||
"""Check if SchedulerOutput is correct after remote KV cache hit."""
|
||||
|
||||
# We should inject the kv_connector_metadata.
|
||||
assert len(output.kv_connector_metadata.requests) == num_requests
|
||||
|
||||
@@ -2148,7 +2153,6 @@ def _assert_right_kv_cache_manager(
|
||||
num_total_blocks: int,
|
||||
):
|
||||
"""Check whether KVCacheManager is correct after allocate."""
|
||||
|
||||
# Make sure the request stats are right.
|
||||
EXPECTED_TOTAL_BLOCKS = num_tokens // block_size
|
||||
for req in requests:
|
||||
@@ -2179,7 +2183,6 @@ def _step_until_done(
|
||||
model_runner_output: ModelRunnerOutput,
|
||||
):
|
||||
"""Loop over schedule(), update_from_output() until finished."""
|
||||
|
||||
all_finished = False
|
||||
_ = scheduler.update_from_output(output, model_runner_output)
|
||||
while not all_finished:
|
||||
@@ -2207,7 +2210,6 @@ def _num_waiting_requests(scheduler: Scheduler) -> int:
|
||||
|
||||
def _step_until_kv_transfer_finished(scheduler: Scheduler, req_ids: list[str]):
|
||||
"""Cycle requests through a KV transfer cycle."""
|
||||
|
||||
# Requests should first transition to WAITING_FOR_REMOTE_KVS
|
||||
output = scheduler.schedule()
|
||||
assert _num_waiting_requests(scheduler) == len(req_ids)
|
||||
@@ -2291,11 +2293,9 @@ def test_has_sync_kv_loads(
|
||||
|
||||
@pytest.mark.parametrize("is_async", [False, True])
|
||||
def test_kv_connector_basic(is_async: bool):
|
||||
"""
|
||||
Test whether Scheduler with KVConnector schedules tokens, allocates
|
||||
"""Test whether Scheduler with KVConnector schedules tokens, allocates
|
||||
memory, and cleans up requests as expected under normal operation.
|
||||
"""
|
||||
|
||||
# Setup Scheduler.
|
||||
BLOCK_SIZE = 16
|
||||
NUM_MATCHED_NEW_TOKENS = BLOCK_SIZE * 2
|
||||
@@ -2420,11 +2420,9 @@ def test_kv_connector_basic(is_async: bool):
|
||||
@pytest.mark.parametrize("is_async", [False, True])
|
||||
@pytest.mark.parametrize("local_cache_hits", [False, True])
|
||||
def test_external_prefix_cache_metrics(is_async: bool, local_cache_hits: bool):
|
||||
"""
|
||||
Verify connector prefix cache metrics are updated
|
||||
"""Verify connector prefix cache metrics are updated
|
||||
correctly when the scheduler processes requests with KV connector hits.
|
||||
"""
|
||||
|
||||
BLOCK_SIZE = 16
|
||||
if local_cache_hits:
|
||||
NUM_MATCHED_NEW_TOKENS = BLOCK_SIZE * 2 # 32 tokens
|
||||
@@ -2540,11 +2538,9 @@ def test_external_prefix_cache_metrics(is_async: bool, local_cache_hits: bool):
|
||||
"use_ec_connector, ec_role", [(False, None), (True, "ec_consumer")]
|
||||
)
|
||||
def test_kv_connector_unable_to_allocate(use_ec_connector, ec_role):
|
||||
"""
|
||||
Test whether scheduler with KVConnector is able to handle
|
||||
"""Test whether scheduler with KVConnector is able to handle
|
||||
unable to allocate (run out of blocks in allocate_slots().
|
||||
"""
|
||||
|
||||
# Setup Scheduler With Mock External Cache Hit.
|
||||
BLOCK_SIZE = 4
|
||||
NUM_BLOCKS = 10
|
||||
@@ -2627,11 +2623,9 @@ def test_kv_connector_unable_to_allocate(use_ec_connector, ec_role):
|
||||
def test_kv_connector_handles_preemption(
|
||||
is_async, use_ec_connector, ec_role, use_v2_model_runner
|
||||
):
|
||||
"""
|
||||
Test whether scheduler with KVConnector is able to handle
|
||||
"""Test whether scheduler with KVConnector is able to handle
|
||||
unable to allocate (run out of blocks in allocate_slots().
|
||||
"""
|
||||
|
||||
# Setup Scheduler With Mock External Cache Hit.
|
||||
BLOCK_SIZE = 2
|
||||
# NOTE: there is 1 null block, so this is 6 blocks.
|
||||
@@ -2842,7 +2836,6 @@ def assert_scheduler_empty(scheduler: Scheduler):
|
||||
|
||||
def test_memory_leak():
|
||||
"""Test that we do not have a memory leak."""
|
||||
|
||||
scheduler = create_scheduler(enable_prefix_caching=True)
|
||||
|
||||
NUM_REQUESTS = 5
|
||||
@@ -2899,6 +2892,7 @@ def create_scheduler_with_priority(
|
||||
|
||||
Returns:
|
||||
{class}`Scheduler` instance with priority scheduling
|
||||
|
||||
"""
|
||||
model_config = ModelConfig(
|
||||
model=model,
|
||||
@@ -3100,7 +3094,8 @@ def create_requests_with_priority(
|
||||
|
||||
def test_priority_scheduling_basic_ordering():
|
||||
"""Test that requests are scheduled in priority order
|
||||
(lower value = higher priority)."""
|
||||
(lower value = higher priority).
|
||||
"""
|
||||
scheduler = create_scheduler_with_priority()
|
||||
|
||||
# Create requests with different priorities
|
||||
@@ -3129,7 +3124,8 @@ def test_priority_scheduling_basic_ordering():
|
||||
|
||||
def test_priority_scheduling_arrival_time_tiebreaker():
|
||||
"""Test that arrival time is used
|
||||
as tiebreaker when priorities are equal."""
|
||||
as tiebreaker when priorities are equal.
|
||||
"""
|
||||
scheduler = create_scheduler_with_priority()
|
||||
|
||||
# Create requests with same priority but different arrival times
|
||||
@@ -3286,7 +3282,8 @@ def test_priority_scheduling_preemption():
|
||||
|
||||
def test_priority_scheduling_no_preemption_when_space_available():
|
||||
"""Test that preemption doesn't happen
|
||||
when there's space for new requests."""
|
||||
when there's space for new requests.
|
||||
"""
|
||||
scheduler = create_scheduler_with_priority(
|
||||
max_num_seqs=3, # Allow 3 concurrent requests
|
||||
max_num_batched_tokens=200, # Sufficient token budget
|
||||
@@ -3339,7 +3336,8 @@ def test_priority_scheduling_no_preemption_when_space_available():
|
||||
|
||||
def test_priority_scheduling_preemption_victim_selection():
|
||||
"""Test that the correct victim is selected for
|
||||
preemption based on priority and arrival time."""
|
||||
preemption based on priority and arrival time.
|
||||
"""
|
||||
# This test verifies the priority-based victim selection logic
|
||||
# by checking the waiting queue order after adding requests with different
|
||||
# priorities
|
||||
@@ -3459,7 +3457,8 @@ def test_priority_scheduling_waiting_queue_order():
|
||||
|
||||
def test_priority_scheduling_fcfs_fallback():
|
||||
"""Test that FCFS behavior is maintained when all
|
||||
requests have same priority."""
|
||||
requests have same priority.
|
||||
"""
|
||||
scheduler = create_scheduler_with_priority()
|
||||
|
||||
# Create requests with same priority but different arrival times
|
||||
@@ -3528,7 +3527,8 @@ def test_priority_scheduling_with_limited_slots():
|
||||
|
||||
def test_priority_scheduling_heap_property():
|
||||
"""Test that the waiting queue maintains heap
|
||||
property for priority scheduling."""
|
||||
property for priority scheduling.
|
||||
"""
|
||||
scheduler = create_scheduler_with_priority(
|
||||
max_num_seqs=1, # Only one request can run at a time
|
||||
)
|
||||
@@ -3753,7 +3753,8 @@ def test_priority_scheduling_preemption_and_resumption_when_out_of_kv(
|
||||
use_ec_connector, ec_role, use_v2_model_runner
|
||||
):
|
||||
"""Test that priority scheduling preempts lower priority requests
|
||||
when out of KV cache space."""
|
||||
when out of KV cache space.
|
||||
"""
|
||||
# Create scheduler with very limited memory to force preemption
|
||||
scheduler = create_scheduler_with_priority(
|
||||
max_num_seqs=2, # Allow multiple requests
|
||||
@@ -3898,7 +3899,8 @@ def test_chunked_prefill_disabled_for_encoder_decoder(
|
||||
enable_chunked_prefill: bool, is_encoder_decoder: bool, expect_enabled: bool
|
||||
) -> None:
|
||||
"""Validate that chunked prefill is appropriately disabled for
|
||||
encoder-decoder models."""
|
||||
encoder-decoder models.
|
||||
"""
|
||||
scheduler_config = SchedulerConfig(
|
||||
enable_chunked_prefill=enable_chunked_prefill,
|
||||
is_encoder_decoder=is_encoder_decoder,
|
||||
@@ -3927,7 +3929,8 @@ def _validate_chunked_prefill_settings_for_encoder_decoder(
|
||||
scheduler_config: SchedulerConfig, is_encoder_decoder: bool, expect_enabled: bool
|
||||
) -> None:
|
||||
"""Validate chunked prefill settings in the scheduler config for
|
||||
encoder-decoder models."""
|
||||
encoder-decoder models.
|
||||
"""
|
||||
assert scheduler_config.enable_chunked_prefill is expect_enabled
|
||||
if is_encoder_decoder:
|
||||
# Encoder-decoder models should automatically disable chunked multimodal
|
||||
@@ -3987,7 +3990,7 @@ def _assert_right_ec_connector_metadata(
|
||||
output: SchedulerOutput,
|
||||
mm_features_list: list[MultiModalFeatureSpec],
|
||||
):
|
||||
"""Verify that ECConnector metadata EXACTLY matches the input MM data"""
|
||||
"""Verify that ECConnector metadata EXACTLY matches the input MM data."""
|
||||
# Get the connector metadata
|
||||
metadata = output.ec_connector_metadata
|
||||
|
||||
@@ -4017,8 +4020,8 @@ def _assert_right_encoder_inputs(
|
||||
):
|
||||
"""Verify that requests/mm_hashes should (not) in scheduled encoder input
|
||||
If check_exist is False, this function returns True
|
||||
if requests are NOT in encoder inputs"""
|
||||
|
||||
if requests are NOT in encoder inputs
|
||||
"""
|
||||
# Get the scheduled encoder inputs
|
||||
# NOTE: scheduled_encoder_inputs is a dictionary with request id as key
|
||||
scheduled_encoder_inputs = output.scheduled_encoder_inputs
|
||||
@@ -4179,7 +4182,8 @@ def test_ec_connector_text_only_request(use_kv_connector):
|
||||
@pytest.mark.parametrize("use_kv_connector", [False, True])
|
||||
def test_ec_connector_cache_hit_external_load(use_kv_connector):
|
||||
"""Test ec_consumer loads from external cache when hit.
|
||||
A normal basic operation for EPD disaggrgation"""
|
||||
A normal basic operation for EPD disaggrgation
|
||||
"""
|
||||
scheduler = create_scheduler(
|
||||
model="llava-hf/llava-1.5-7b-hf",
|
||||
enable_prefix_caching=True,
|
||||
@@ -4527,11 +4531,9 @@ def test_ec_connector_schedule_multiple_requests(cache_exist, use_kv_connector):
|
||||
|
||||
@pytest.mark.parametrize("use_kv_connector", [False, True])
|
||||
def test_ec_connector_unable_to_allocate(use_kv_connector):
|
||||
"""
|
||||
Test whether scheduler with ECConnector is able to handle
|
||||
"""Test whether scheduler with ECConnector is able to handle
|
||||
unable to allocate (run out of blocks).
|
||||
"""
|
||||
|
||||
# Setup Scheduler With Mock External Cache Hit.
|
||||
BLOCK_SIZE = 4
|
||||
NUM_BLOCKS = 10
|
||||
@@ -4628,7 +4630,8 @@ def test_priority_scheduling_ec_connector_preemption_and_resumption(
|
||||
cache_exist, use_kv_connector
|
||||
):
|
||||
"""Test that priority scheduling preempts lower priority requests
|
||||
when out of KV cache space."""
|
||||
when out of KV cache space.
|
||||
"""
|
||||
# Create scheduler with very limited memory to force preemption
|
||||
scheduler = create_scheduler_with_priority(
|
||||
model="llava-hf/llava-1.5-7b-hf",
|
||||
@@ -4860,8 +4863,7 @@ def test_priority_scheduling_ec_connector_preemption_and_resumption(
|
||||
|
||||
@pytest.mark.parametrize("use_kv_connector", [False, True])
|
||||
def test_ec_connector_allocate_encoder_tokens_with_external_load(use_kv_connector):
|
||||
"""
|
||||
Scenario:
|
||||
"""Scenario:
|
||||
- Encoder cache size: 32
|
||||
- Request A: 1 feature (12 tokens) → NOT cached remotely.
|
||||
- Request B: 3 features (3 x 10 tokens) → ALL cached remotely.
|
||||
@@ -5298,7 +5300,8 @@ def test_scheduler_kv_connector_stats():
|
||||
|
||||
def test_ec_connector_update_connector_output_called():
|
||||
"""Test that worker-side EC connector output is forwarded to the
|
||||
EC connector's update_connector_output hook."""
|
||||
EC connector's update_connector_output hook.
|
||||
"""
|
||||
scheduler = create_scheduler(
|
||||
model="llava-hf/llava-1.5-7b-hf",
|
||||
use_ec_connector=True,
|
||||
@@ -5503,7 +5506,8 @@ def test_variable_length_cross_attn_block_allocation():
|
||||
|
||||
def test_cross_attn_blocks_not_over_allocated():
|
||||
"""Test that cross-attention blocks are not over-allocated compared to
|
||||
what each request actually needs."""
|
||||
what each request actually needs.
|
||||
"""
|
||||
from math import ceil
|
||||
|
||||
block_size = 16
|
||||
@@ -5599,7 +5603,8 @@ def test_cross_attn_blocks_not_under_allocated():
|
||||
|
||||
def test_cross_attn_zero_blocks_without_encoder_inputs():
|
||||
"""Test that requests without encoder inputs get zero cross-attention
|
||||
blocks, even when the scheduler is configured for encoder-decoder."""
|
||||
blocks, even when the scheduler is configured for encoder-decoder.
|
||||
"""
|
||||
block_size = 16
|
||||
scheduler = _create_encoder_decoder_scheduler(block_size=block_size)
|
||||
|
||||
@@ -5704,7 +5709,8 @@ def test_free_encoder_inputs_respects_unconfirmed_placeholders():
|
||||
evicted entry and crashes the engine with "Encoder cache miss". The
|
||||
scheduler must retain the input until the *confirmed* progress
|
||||
(num_computed_tokens - num_output_placeholders) passes the range end, so
|
||||
that no pending rejection can rewind into the range."""
|
||||
that no pending rejection can rewind into the range.
|
||||
"""
|
||||
scheduler = create_scheduler(
|
||||
model="llava-hf/llava-1.5-7b-hf",
|
||||
num_speculative_tokens=3,
|
||||
@@ -5815,7 +5821,8 @@ def test_free_encoder_inputs_defers_for_eagle_lookahead():
|
||||
position so the drafter's +1 look-ahead mm-embedding gather (which reads one
|
||||
position past the target's computed range) still finds it cached. This is
|
||||
the primary mechanism that prevents the drafter "Encoder cache miss"; the
|
||||
worker-side token-embedding fallback is only a backstop."""
|
||||
worker-side token-embedding fallback is only a backstop.
|
||||
"""
|
||||
scheduler = create_scheduler(model="llava-hf/llava-1.5-7b-hf")
|
||||
# create_scheduler only builds ngram spec configs; force the eagle path that
|
||||
# _free_encoder_inputs keys off (its read-ahead deferral).
|
||||
@@ -5845,7 +5852,8 @@ def test_free_encoder_inputs_defers_for_eagle_lookahead():
|
||||
|
||||
def test_free_encoder_inputs_unchanged_without_spec_decode():
|
||||
"""Without speculative decoding, encoder inputs are freed as soon as
|
||||
num_computed_tokens passes the placeholder range, as before."""
|
||||
num_computed_tokens passes the placeholder range, as before.
|
||||
"""
|
||||
scheduler = create_scheduler(model="llava-hf/llava-1.5-7b-hf")
|
||||
mm_positions = [[PlaceholderRange(offset=50, length=100)]]
|
||||
request = create_requests(
|
||||
@@ -5876,7 +5884,8 @@ def test_encoder_cache_retained_across_preemption_and_resume():
|
||||
encoder input must pull the still-cached entry back out of `freeable`
|
||||
without scheduling a recompute, keeping the scheduler and worker
|
||||
consistent. The spec-rollback retention margin does not gate this path,
|
||||
so it is covered separately here."""
|
||||
so it is covered separately here.
|
||||
"""
|
||||
scheduler = create_scheduler(
|
||||
model="llava-hf/llava-1.5-7b-hf",
|
||||
num_speculative_tokens=3,
|
||||
@@ -5920,7 +5929,8 @@ def test_encoder_cache_recomputed_when_evicted_during_preemption():
|
||||
pressure before it resumes, the scheduler reports the mm_hash as freed
|
||||
(so the worker drops it) and a resume must schedule a recompute rather
|
||||
than assume the worker still holds it. check_and_update_cache must
|
||||
return False so the encoder input is re-scheduled."""
|
||||
return False so the encoder input is re-scheduled.
|
||||
"""
|
||||
scheduler = create_scheduler(
|
||||
model="llava-hf/llava-1.5-7b-hf",
|
||||
num_speculative_tokens=3,
|
||||
@@ -6571,7 +6581,8 @@ def _decode_ready_request(scheduler):
|
||||
|
||||
def test_update_draft_token_ids_strips_ngram_padding():
|
||||
"""ngram_gpu pads unfilled draft slots with -1; manager validation must
|
||||
strip them before grammar.validate_tokens (which otherwise raises)."""
|
||||
strip them before grammar.validate_tokens (which otherwise raises).
|
||||
"""
|
||||
scheduler = create_scheduler(num_speculative_tokens=4)
|
||||
request = _decode_ready_request(scheduler)
|
||||
|
||||
@@ -6590,7 +6601,8 @@ def test_update_draft_token_ids_strips_ngram_padding():
|
||||
|
||||
def test_update_draft_token_ids_in_output_strips_padding():
|
||||
"""Same guard on the output path; the -1 pad-back for the rejected count
|
||||
is preserved (only the input to manager validation is stripped)."""
|
||||
is preserved (only the input to manager validation is stripped).
|
||||
"""
|
||||
scheduler = create_scheduler(num_speculative_tokens=4)
|
||||
request = _decode_ready_request(scheduler)
|
||||
|
||||
|
||||
@@ -97,7 +97,8 @@ def test_num_in_flight_tokens_accounting():
|
||||
|
||||
def test_swa_free_waits_for_in_flight_step():
|
||||
"""Async: out-of-window blocks stay allocated until the step that still
|
||||
reads them has been processed."""
|
||||
reads them has been processed.
|
||||
"""
|
||||
scheduler = _create_swa_scheduler(async_scheduling=True)
|
||||
request = create_requests(
|
||||
num_requests=1, num_tokens=NUM_PROMPT_TOKENS, block_size=BLOCK_SIZE
|
||||
@@ -132,7 +133,8 @@ def test_swa_free_waits_for_in_flight_step():
|
||||
|
||||
def test_swa_free_immediate_when_sync():
|
||||
"""Sync: no in-flight step at schedule time, frees happen at the first
|
||||
decode allocation as before."""
|
||||
decode allocation as before.
|
||||
"""
|
||||
scheduler = _create_swa_scheduler(async_scheduling=False)
|
||||
request = create_requests(
|
||||
num_requests=1, num_tokens=NUM_PROMPT_TOKENS, block_size=BLOCK_SIZE
|
||||
@@ -171,7 +173,8 @@ def test_swa_admission_cap_accounts_for_overlapping_batches():
|
||||
def test_chunked_local_free_waits_for_in_flight_step():
|
||||
"""Chunked-local attention frees whole chunks left of the current one, and
|
||||
is exposed to the same load-WAR: with async scheduling those chunks must
|
||||
stay allocated until the in-flight step that still reads them settles."""
|
||||
stay allocated until the in-flight step that still reads them settles.
|
||||
"""
|
||||
scheduler = _create_chunked_scheduler(async_scheduling=True)
|
||||
request = create_requests(
|
||||
num_requests=1, num_tokens=NUM_PROMPT_TOKENS, block_size=BLOCK_SIZE
|
||||
@@ -226,7 +229,8 @@ def test_connector_finish_frees_on_settled_basis():
|
||||
is handed to a KV connector (simple CPU offload / NIXL store), must use the
|
||||
same processed-token basis. Otherwise the connector reads/hands off a block
|
||||
the still-in-flight step is optimistically counted as done with, which is
|
||||
the load-path WAR this fix closes."""
|
||||
the load-path WAR this fix closes.
|
||||
"""
|
||||
scheduler = create_scheduler(
|
||||
block_size=BLOCK_SIZE,
|
||||
async_scheduling=True,
|
||||
|
||||
@@ -89,6 +89,7 @@ def create_scheduler(
|
||||
|
||||
Returns:
|
||||
{class}`Scheduler` instance
|
||||
|
||||
"""
|
||||
model_config = ModelConfig(
|
||||
model=model,
|
||||
|
||||
Reference in New Issue
Block a user