Add pydocstyle to the ruff rules (#52136)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: Harry Mellor <hej.mellor@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
804e5377aa
commit
8c1557a79c
@@ -151,6 +151,7 @@ def create_and_prepopulate_kv_cache(
|
||||
Returns:
|
||||
A 4D tensor in logical ``(num_blocks, num_kv_heads, block_size,
|
||||
2 * head_size)`` order with strides determined by ``layout``.
|
||||
|
||||
"""
|
||||
batch_size = len(k_contexts)
|
||||
seq_lens = common_attn_metadata.seq_lens.cpu()
|
||||
@@ -286,7 +287,6 @@ def run_attention_backend(
|
||||
use_cuda_graph: bool = False,
|
||||
) -> torch.Tensor:
|
||||
"""Run attention computation using the specified backend's AttentionImpl."""
|
||||
|
||||
use_direct_block_mask = is_torch_equal_or_newer("2.9.0.dev0")
|
||||
if backend == "FLEX_ATTENTION_SLOW":
|
||||
use_direct_block_mask = False
|
||||
@@ -414,8 +414,7 @@ def _test_backend_correctness(
|
||||
max_num_seqs: int | None = None,
|
||||
max_num_batched_tokens: int | None = None,
|
||||
):
|
||||
"""
|
||||
Test that all backends produce similar outputs to a reference implementation
|
||||
"""Test that all backends produce similar outputs to a reference implementation
|
||||
using FlexAttention or an explicit attention-sink reference.
|
||||
|
||||
This test works by:
|
||||
@@ -1393,7 +1392,8 @@ def test_non_causal_backend_correctness(
|
||||
model: str,
|
||||
):
|
||||
"""Test backend's correctness with non-causal (bidirectional) decoder
|
||||
attention, as used by DFlash speculative decoding."""
|
||||
attention, as used by DFlash speculative decoding.
|
||||
"""
|
||||
|
||||
def bidirectional_mask_mod(
|
||||
b: torch.Tensor,
|
||||
|
||||
@@ -115,7 +115,8 @@ def test_mamba_layers_have_unified_interface(
|
||||
layer_class, expected_backend, expected_mamba_type
|
||||
):
|
||||
"""Test that all Mamba layers have the unified get_attn_backend
|
||||
interface."""
|
||||
interface.
|
||||
"""
|
||||
assert hasattr(layer_class, "get_attn_backend"), (
|
||||
f"{layer_class.__name__} should have get_attn_backend method"
|
||||
)
|
||||
|
||||
@@ -20,12 +20,12 @@ from vllm.v1.worker.ubatch_utils import (
|
||||
|
||||
@pytest.fixture
|
||||
def sample_query_start_loc():
|
||||
"""Sample query_start_loc tensor for testing"""
|
||||
"""Sample query_start_loc tensor for testing."""
|
||||
return torch.tensor([0, 5, 12, 20, 35, 50])
|
||||
|
||||
|
||||
def test_basic_slice_middle(sample_query_start_loc):
|
||||
"""Test slicing from middle of tensor"""
|
||||
"""Test slicing from middle of tensor."""
|
||||
req_slice = slice(1, 3) # slice from index 1 to 3
|
||||
result = slice_query_start_locs(sample_query_start_loc, req_slice)
|
||||
|
||||
@@ -34,7 +34,7 @@ def test_basic_slice_middle(sample_query_start_loc):
|
||||
|
||||
|
||||
def test_slice_from_beginning(sample_query_start_loc):
|
||||
"""Test slicing from the beginning of tensor"""
|
||||
"""Test slicing from the beginning of tensor."""
|
||||
req_slice = slice(0, 2) # slice from index 0 to 2
|
||||
result = slice_query_start_locs(sample_query_start_loc, req_slice)
|
||||
|
||||
@@ -43,7 +43,7 @@ def test_slice_from_beginning(sample_query_start_loc):
|
||||
|
||||
|
||||
def test_slice_to_end(sample_query_start_loc):
|
||||
"""Test slicing to the end of tensor"""
|
||||
"""Test slicing to the end of tensor."""
|
||||
req_slice = slice(3, 5) # slice from index 3 to 5 (last index)
|
||||
result = slice_query_start_locs(sample_query_start_loc, req_slice)
|
||||
|
||||
@@ -52,7 +52,7 @@ def test_slice_to_end(sample_query_start_loc):
|
||||
|
||||
|
||||
def test_single_element_slice(sample_query_start_loc):
|
||||
"""Test slice that results in single element"""
|
||||
"""Test slice that results in single element."""
|
||||
req_slice = slice(2, 3) # slice from index 2 to 3
|
||||
result = slice_query_start_locs(sample_query_start_loc, req_slice)
|
||||
|
||||
@@ -61,7 +61,7 @@ def test_single_element_slice(sample_query_start_loc):
|
||||
|
||||
|
||||
def test_full_tensor_slice(sample_query_start_loc):
|
||||
"""Test slicing the entire tensor"""
|
||||
"""Test slicing the entire tensor."""
|
||||
req_slice = slice(0, 5) # slice entire tensor
|
||||
result = slice_query_start_locs(sample_query_start_loc, req_slice)
|
||||
|
||||
@@ -80,7 +80,7 @@ def test_slice_bounds_edge_cases(sample_query_start_loc):
|
||||
|
||||
@pytest.fixture
|
||||
def small_decode_metadata():
|
||||
"""Create metadata for small decode batch"""
|
||||
"""Create metadata for small decode batch."""
|
||||
batch_spec = BATCH_SPECS["small_decode"]
|
||||
device = torch.device("cpu")
|
||||
return create_common_attn_metadata(batch_spec, block_size=16, device=device)
|
||||
@@ -88,7 +88,7 @@ def small_decode_metadata():
|
||||
|
||||
@pytest.fixture
|
||||
def large_decode_metadata():
|
||||
"""Create metadata for small decode batch"""
|
||||
"""Create metadata for small decode batch."""
|
||||
batch_spec = BATCH_SPECS["large_decode"]
|
||||
device = torch.device("cpu")
|
||||
return create_common_attn_metadata(batch_spec, block_size=16, device=device)
|
||||
@@ -96,7 +96,7 @@ def large_decode_metadata():
|
||||
|
||||
@pytest.fixture
|
||||
def mixed_small_metadata():
|
||||
"""Create metadata for mixed small batch"""
|
||||
"""Create metadata for mixed small batch."""
|
||||
batch_spec = BATCH_SPECS["mixed_small"]
|
||||
device = torch.device("cpu")
|
||||
return create_common_attn_metadata(batch_spec, block_size=16, device=device)
|
||||
@@ -104,7 +104,7 @@ def mixed_small_metadata():
|
||||
|
||||
# Tests for _make_metadata_with_slice
|
||||
def test_make_metadata_with_slice_decode_batch(small_decode_metadata):
|
||||
"""Test slicing decode batch metadata"""
|
||||
"""Test slicing decode batch metadata."""
|
||||
# Split first request only
|
||||
ubatch_slice = UBatchSlice(slice(0, 1), slice(0, 1))
|
||||
|
||||
@@ -119,7 +119,7 @@ def test_make_metadata_with_slice_decode_batch(small_decode_metadata):
|
||||
|
||||
|
||||
def test_make_metadata_with_slice_mixed_batch(mixed_small_metadata):
|
||||
"""Test slicing mixed batch metadata"""
|
||||
"""Test slicing mixed batch metadata."""
|
||||
ubatch_slice = UBatchSlice(slice(1, 3), slice(1, 7)) # Requests 1-3, tokens 1-7
|
||||
|
||||
result = _make_metadata_with_slice(ubatch_slice, mixed_small_metadata)
|
||||
@@ -132,7 +132,7 @@ def test_make_metadata_with_slice_mixed_batch(mixed_small_metadata):
|
||||
|
||||
|
||||
def test_split_attn_metadata_decode_batch(large_decode_metadata):
|
||||
"""Test splitting decode batch into two equal parts"""
|
||||
"""Test splitting decode batch into two equal parts."""
|
||||
num_tokens = large_decode_metadata.num_reqs
|
||||
mid_point = num_tokens // 2
|
||||
ubatch_slices = [
|
||||
@@ -164,7 +164,8 @@ def apply_split_decodes_and_prefills(
|
||||
treat_short_extends_as_decodes: bool = True,
|
||||
):
|
||||
"""Helper function to apply split_decodes_and_prefills and return
|
||||
the results."""
|
||||
the results.
|
||||
"""
|
||||
device = torch.device("cpu")
|
||||
seq_lens = [10 * (i + 1) for i in range(len(query_lens))]
|
||||
common_metadata = create_common_attn_metadata(
|
||||
@@ -297,7 +298,7 @@ def test_split_decodes_and_prefills_uniform_mixed_batch_non_uniform_decodes():
|
||||
|
||||
|
||||
def test_split_decodes_and_prefills_uniform_padded_batch_all_same():
|
||||
"""uniform batch where all query lengths are identical with 0 length padded reqs."""
|
||||
"""Uniform batch where all query lengths are identical with 0 length padded reqs."""
|
||||
# All query lengths are 2, with decode_threshold=3 (so 2 <= 3)
|
||||
# This triggers the padded uniform path at line 891
|
||||
query_lens = [2, 2, 2, 0]
|
||||
@@ -324,7 +325,7 @@ def test_split_decodes_and_prefills_uniform_padded_batch_all_same():
|
||||
def test_prefill_split_across_ubatches(
|
||||
seq_lens, query_lens, split_point, expected_first_reqs, expected_second_reqs
|
||||
):
|
||||
"""Test splitting a prefill across ubatches"""
|
||||
"""Test splitting a prefill across ubatches."""
|
||||
import numpy as np
|
||||
|
||||
device = torch.device("cpu")
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""The empty-shard mask fused into the DCP A2A pack kernel must be bit-exact
|
||||
with the eager ``mask_dcp_empty_shards_`` pass it replaces."""
|
||||
with the eager ``mask_dcp_empty_shards_`` pass it replaces.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
@@ -495,7 +495,8 @@ def test_dspark_noncausal_sparse_mla_matches_sdpa(
|
||||
num_heads,
|
||||
):
|
||||
"""Non-causal (window ∪ block, future-pointing) per-token indices must match
|
||||
a dense SDPA reference over the same indices, for both sparse-MLA backends."""
|
||||
a dense SDPA reference over the same indices, for both sparse-MLA backends.
|
||||
"""
|
||||
_skip_if_backend_unavailable(backend_cls, kv_cache_dtype, block_size)
|
||||
|
||||
window, block, topk_width = _DSPARK_CONFIGS[config_name]
|
||||
|
||||
@@ -231,7 +231,8 @@ def test_plan_uses_state_params(monkeypatch):
|
||||
|
||||
def test_kv_lens_host_formula():
|
||||
"""Per-row host lengths: context == position + 1; capped at
|
||||
index_topk + trailing-pool remainder past the sparse threshold."""
|
||||
index_topk + trailing-pool remainder past the sparse threshold.
|
||||
"""
|
||||
builder = object.__new__(FlashInferMLASparseSM90Builder)
|
||||
builder._index_topk = 2048
|
||||
builder._index_kpool = 4
|
||||
|
||||
@@ -187,7 +187,8 @@ def test_gdn_build_classification(test_case: GDNBuildTestCase):
|
||||
def test_has_initial_state_after_reclassification():
|
||||
"""After reclassification, num_prefills > 0 so the prefill kernel path
|
||||
should compute has_initial_state. For the reclassified request with
|
||||
context_lens > 0, the corresponding entry must be True."""
|
||||
context_lens > 0, the corresponding entry must be True.
|
||||
"""
|
||||
builder = _create_gdn_builder(num_speculative_tokens=2)
|
||||
batch = BatchSpec(seq_lens=[65, 20], query_lens=[1, 3])
|
||||
meta = _build(builder, batch, num_decode_draft_tokens=[-1, 2])
|
||||
|
||||
@@ -56,7 +56,8 @@ def _ref_stable_topk_from_candidates_fp64(
|
||||
) -> torch.Tensor:
|
||||
"""Pure-PyTorch reference for the CuteDSL stable-topk selector order
|
||||
(score desc, then lowest global token id). Selects the same SET as the
|
||||
kernel; only the set is compared in tests."""
|
||||
kernel; only the set is compared in tests.
|
||||
"""
|
||||
num_rows, num_candidates = candidate_scores.shape
|
||||
device = candidate_scores.device
|
||||
select_k = min(k, num_candidates)
|
||||
@@ -121,7 +122,8 @@ def _dcp_lse_merge(
|
||||
class _FakeDCPGroup:
|
||||
"""Single-process stand-in: ``all_gather`` returns the pre-built
|
||||
concatenation of every rank's packed ``(score, global_id)`` candidates,
|
||||
mirroring the one packed all-gather the merge issues."""
|
||||
mirroring the one packed all-gather the merge issues.
|
||||
"""
|
||||
|
||||
def __init__(self, gathered_packed: torch.Tensor) -> None:
|
||||
self.gathered_packed = gathered_packed
|
||||
@@ -207,7 +209,8 @@ def _merge_local_topks_global_with_fake_dcp(
|
||||
"""Run ``_merge_dcp_topk_global`` for every rank against a faked
|
||||
all-gather and return each rank's global-index result (all ranks should
|
||||
agree). The fake pre-builds the packed candidate concatenation the merge's
|
||||
single ``all_gather`` would return."""
|
||||
single ``all_gather`` would return.
|
||||
"""
|
||||
# The merge is now CuteDSL-only (no PyTorch fallback), so it runs the real
|
||||
# Triton pack + CuteDSL selector kernels even behind the faked all-gather.
|
||||
if not current_platform.is_cuda() or not has_cutedsl():
|
||||
@@ -709,7 +712,8 @@ def test_dcp_filter_compaction_matches_reference(
|
||||
"""In-kernel compaction must, for every row, produce exactly the rank-owned
|
||||
physical slots packed into [0, valid_count) with -1 in the tail -- the same
|
||||
SET a reference filter + sort/gather produces. Uses wide rows (> BLOCK_N
|
||||
valid slots), with interior -1 gaps."""
|
||||
valid slots), with interior -1 gaps.
|
||||
"""
|
||||
device = torch.device("cuda")
|
||||
torch.manual_seed(7)
|
||||
dcp_size = 2
|
||||
@@ -867,7 +871,8 @@ def test_decode_topk_pads_surplus_with_negative_one():
|
||||
so a non-(-1) pad would be silently attended. This is the common case under
|
||||
DCP, where each rank's local seq_len = global / world is usually << topk.
|
||||
Checked on the *set* of valid indices (the kernel may order them by score,
|
||||
not position)."""
|
||||
not position).
|
||||
"""
|
||||
torch.manual_seed(0)
|
||||
device = torch.device("cuda")
|
||||
topk = 8
|
||||
@@ -913,7 +918,8 @@ def test_sparse_decode_dcp_short_context_matches_non_dcp():
|
||||
"""End-to-end DCP decode where the global seq_len < topk (so every rank's
|
||||
local top-k is surplus-padded). Exercises the kernel surplus -> merge mask
|
||||
-> global top-k -> physical localize -> LSE merge chain for the common
|
||||
short-context decode case, vs the non-DCP reference."""
|
||||
short-context decode case, vs the non-DCP reference.
|
||||
"""
|
||||
torch.manual_seed(4)
|
||||
device = torch.device("cuda")
|
||||
world = 2
|
||||
|
||||
@@ -58,7 +58,8 @@ def test_tail_backend_layout_matches_kernel_pointer_arithmetic():
|
||||
def make_tail_block_table(own_blocks, width=64):
|
||||
"""Tail-group block table as BlockTables produces it: column 0 holds the
|
||||
request's single KpoolTailManager block, the remaining columns are never
|
||||
written and stay zero."""
|
||||
written and stay zero.
|
||||
"""
|
||||
bt = torch.zeros(len(own_blocks), width, dtype=torch.int32)
|
||||
bt[:, 0] = torch.tensor(own_blocks, dtype=torch.int32)
|
||||
return bt
|
||||
@@ -66,7 +67,8 @@ def make_tail_block_table(own_blocks, width=64):
|
||||
|
||||
def legacy_generic_tail_slots(block_table, query_start_loc, positions):
|
||||
"""Reference of the generic ``_compute_slot_mappings_kernel`` arithmetic
|
||||
(block_table.py:305-313) applied to the tail group's table."""
|
||||
(block_table.py:305-313) applied to the tail group's table.
|
||||
"""
|
||||
slots = []
|
||||
for req in range(block_table.shape[0]):
|
||||
for i in range(query_start_loc[req], query_start_loc[req + 1]):
|
||||
@@ -107,7 +109,8 @@ def make_batch(per_req_positions, padded_len=None):
|
||||
|
||||
def test_legacy_generic_mapping_collapses_onto_block_zero():
|
||||
"""The bug: with the manager's 1-column block table, the generic kernel
|
||||
maps every pos >= kpool onto tail block 0, and distinct requests collide."""
|
||||
maps every pos >= kpool onto tail block 0, and distinct requests collide.
|
||||
"""
|
||||
own_blocks = [5, 9]
|
||||
per_req = [list(range(10)), list(range(12))] # prompts of len 10 and 12
|
||||
positions, qsl, _, num_actual, num_reqs = make_batch(per_req)
|
||||
@@ -137,7 +140,8 @@ def test_legacy_generic_mapping_collapses_onto_block_zero():
|
||||
def test_circular_mapping_isolates_requests():
|
||||
"""The fix: every token lands in its own request's block at pos % kpool,
|
||||
and no slot is ever shared by two different requests (slots do recur
|
||||
within a request every kpool positions -- that is the circular design)."""
|
||||
within a request every kpool positions -- that is the circular design).
|
||||
"""
|
||||
own_blocks = [5, 9]
|
||||
per_req = [list(range(10)), list(range(12))]
|
||||
positions, qsl, slot_mapping, num_actual, num_reqs = make_batch(per_req)
|
||||
@@ -165,7 +169,8 @@ def test_circular_mapping_isolates_requests():
|
||||
def test_circular_mapping_matches_generic_for_short_requests(prompt_len):
|
||||
"""For pos < kpool the generic kernel already picks the own block, so the
|
||||
two mappings agree while every position fits the request's first block
|
||||
(single-request behavior is unchanged)."""
|
||||
(single-request behavior is unchanged).
|
||||
"""
|
||||
own_blocks = [7]
|
||||
per_req = [list(range(prompt_len))]
|
||||
positions, qsl, slot_mapping, num_actual, num_reqs = make_batch(per_req)
|
||||
@@ -275,7 +280,8 @@ class TailRingMirror:
|
||||
"""Mirror of _kpool_tail_seed_kernel / _kpool_decode_update_batched_kernel
|
||||
addressing: block = tail_slot // kpool, ring offset = pos % kpool; a pool
|
||||
completing at pos reads ring slots (pool_start + s) % kpool and uses the
|
||||
current token's own K/score for the last member."""
|
||||
current token's own K/score for the last member.
|
||||
"""
|
||||
|
||||
def __init__(self, num_blocks, kpool=KPOOL):
|
||||
self.kpool = kpool
|
||||
@@ -318,7 +324,8 @@ def tail_slot_for(mapping, req, pos, own_block):
|
||||
|
||||
def run_scenario(mapping, interleave):
|
||||
"""Requests A (block 5, prompt len 9) and B (block 9, prompt len 11)
|
||||
decode concurrently; returns A's boundary pool [8, 9, 10, 11]."""
|
||||
decode concurrently; returns A's boundary pool [8, 9, 10, 11].
|
||||
"""
|
||||
ring = TailRingMirror(num_blocks=16)
|
||||
blocks = {"A": 5, "B": 9}
|
||||
prompts = {"A": 9, "B": 11}
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Regression test for https://github.com/vllm-project/vllm/issues/34865
|
||||
"""Regression test for https://github.com/vllm-project/vllm/issues/34865.
|
||||
|
||||
When multiple KV cache groups share the same MambaSpec (as in Nemotron
|
||||
hybrid models), the metadata caching optimization reuses metadata from
|
||||
@@ -29,7 +28,8 @@ def _make_vllm_config(
|
||||
block_size: int | None = None,
|
||||
):
|
||||
"""Create a minimal mock VllmConfig with only the fields the builder
|
||||
accesses, avoiding any model download / HF config inspection."""
|
||||
accesses, avoiding any model download / HF config inspection.
|
||||
"""
|
||||
speculative_config = (
|
||||
SimpleNamespace(
|
||||
num_speculative_tokens=num_speculative_tokens,
|
||||
@@ -75,8 +75,8 @@ def test_mamba_single_token_prompt_runs_as_prefill():
|
||||
def test_update_block_table_copies_block_idx_to_persistent_buffers():
|
||||
"""update_block_table() must write block_idx tensors to the current
|
||||
builder's persistent buffers, not leave them pointing to a different
|
||||
builder's buffers."""
|
||||
|
||||
builder's buffers.
|
||||
"""
|
||||
block_size = 16
|
||||
max_model_len = 256
|
||||
num_reqs = 4
|
||||
@@ -180,8 +180,8 @@ def test_state_indices_tensor_d_includes_num_speculative_blocks():
|
||||
bug 1: with mamba_cache_mode='all' and speculative decoding enabled,
|
||||
the cudagraph buffer for state_indices_tensor_d must allocate the same
|
||||
per-request column count as the runtime block table, which includes
|
||||
num_speculative_blocks trailing scratch columns."""
|
||||
|
||||
num_speculative_blocks trailing scratch columns.
|
||||
"""
|
||||
block_size = 16
|
||||
max_model_len = 256
|
||||
max_num_seqs = 4
|
||||
@@ -214,8 +214,8 @@ def test_block_idx_cudagraph_capture_padded_by_num_reqs():
|
||||
bug 2: with mamba_cache_mode='all' and spec decode, _update_metadata_for
|
||||
_cudagraph_capture must slice block_idx_last_{scheduled,computed}_token
|
||||
by the request count (padded_bs == num_reqs), not by num_decode_tokens.
|
||||
Past num_decodes, the slice must be zero-filled."""
|
||||
|
||||
Past num_decodes, the slice must be zero-filled.
|
||||
"""
|
||||
block_size = 16
|
||||
max_model_len = 256
|
||||
max_num_seqs = 8
|
||||
@@ -295,7 +295,8 @@ def test_block_idx_prev_step_persistent_buffer_allocated():
|
||||
block_idx_last_scheduled_token_prev_step as a persistent buffer with the
|
||||
same shape as the existing block_idx_last_{scheduled,computed}_token
|
||||
buffers, so cudagraph capture records a stable pointer for the prev-step
|
||||
input anchor consumed by mamba_mixer2's input gather."""
|
||||
input anchor consumed by mamba_mixer2's input gather.
|
||||
"""
|
||||
block_size = 16
|
||||
max_model_len = 256
|
||||
max_num_seqs = 8
|
||||
@@ -323,7 +324,8 @@ def test_block_idx_prev_step_persistent_buffer_allocated():
|
||||
|
||||
def test_block_idx_prev_step_persistent_buffer_skipped_without_spec_decode():
|
||||
"""Without spec decode, the prev-step buffer is unused and must not be
|
||||
allocated — the input anchor reduces to last_computed_token."""
|
||||
allocated — the input anchor reduces to last_computed_token.
|
||||
"""
|
||||
block_size = 16
|
||||
max_model_len = 256
|
||||
max_num_seqs = 8
|
||||
@@ -347,7 +349,8 @@ def test_block_idx_prev_step_cudagraph_capture_uses_persistent_buffer():
|
||||
"""_update_metadata_for_cudagraph_capture must copy the prev-step anchor
|
||||
into the builder's persistent buffer (so cudagraph replay reads from the
|
||||
same underlying memory), pad past num_decodes with zero, and return a
|
||||
slice of the persistent buffer in the metadata."""
|
||||
slice of the persistent buffer in the metadata.
|
||||
"""
|
||||
block_size = 16
|
||||
max_model_len = 256
|
||||
max_num_seqs = 8
|
||||
|
||||
@@ -171,7 +171,8 @@ def test_glm5_flashinfer_masked_mha_routing(
|
||||
def test_concat_k_nope_k_pe_matches_torch_cat(qk_rope_head_dim):
|
||||
"""The K concat used by the MLA prefill context loop must equal torch.cat of
|
||||
k_nope with the broadcast k_pe; with no RoPE part it returns k_nope itself
|
||||
instead of allocating and copying."""
|
||||
instead of allocating and copying.
|
||||
"""
|
||||
torch.manual_seed(0)
|
||||
num_tokens, num_heads, qk_nope_head_dim = 5, 4, 256
|
||||
k_nope = torch.randn(num_tokens, num_heads, qk_nope_head_dim, dtype=torch.bfloat16)
|
||||
@@ -515,6 +516,7 @@ def create_and_prepopulate_kv_cache(
|
||||
|
||||
Returns:
|
||||
MLA KV cache tensor
|
||||
|
||||
"""
|
||||
batch_size = len(kv_c_contexts)
|
||||
seq_lens = common_attn_metadata.seq_lens.cpu()
|
||||
@@ -1307,7 +1309,6 @@ def run_attention_backend(
|
||||
chunked_prefill_workspace_size: int | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Run attention computation using the specified backend's AttentionImpl."""
|
||||
|
||||
builder_cls, impl_cls = try_get_attention_backend(backend)
|
||||
|
||||
# Force the prefill backend selection (None means auto-select).
|
||||
@@ -1434,8 +1435,7 @@ def _run_backend_correctness(
|
||||
v_head_dim: int,
|
||||
chunked_prefill_workspace_size: int | None = None,
|
||||
):
|
||||
"""
|
||||
Test that all backends produce similar outputs to a reference implementation
|
||||
"""Test that all backends produce similar outputs to a reference implementation
|
||||
using torch.nn.functional.scaled_dot_product_attention.
|
||||
|
||||
This test works by:
|
||||
@@ -1454,7 +1454,6 @@ def _run_backend_correctness(
|
||||
multiple GPUs. This tests that backends work correctly with different
|
||||
head counts.
|
||||
"""
|
||||
|
||||
# Filter backends to those that support the requested kv_cache_dtype
|
||||
backends_to_test = [
|
||||
b
|
||||
|
||||
@@ -134,7 +134,8 @@ def test_flash_attn_registers_warmup_only_for_fa4(
|
||||
|
||||
def test_flash_attn_prefill_backend_signature_accepts_fused_kwargs():
|
||||
"""run_prefill_new_tokens must accept out/output_scale so the direct
|
||||
(non-**kwargs) call in forward_mha type- and runtime-checks."""
|
||||
(non-**kwargs) call in forward_mha type- and runtime-checks.
|
||||
"""
|
||||
import inspect
|
||||
|
||||
params = inspect.signature(
|
||||
@@ -150,7 +151,8 @@ def test_flash_attn_prefill_backend_signature_accepts_fused_kwargs():
|
||||
|
||||
def test_mla_impl_forward_mha_accepts_output_scale():
|
||||
"""The abstract MLA impl forward_mha must carry output_scale so every
|
||||
override (and the unconditional forward_impl call) stays compatible."""
|
||||
override (and the unconditional forward_impl call) stays compatible.
|
||||
"""
|
||||
import inspect
|
||||
|
||||
from vllm.v1.attention.backend import MLAAttentionImpl
|
||||
@@ -177,7 +179,8 @@ _FUSED_FP8_SKIP = _fused_fp8_skip_reason()
|
||||
def test_fa4_fused_fp8_output_matches_post_quant(default_vllm_config):
|
||||
"""FA4's fused FP8 write (output_scale, flash-attention#135) must match the
|
||||
bf16-attention + standalone static-FP8-quant path it replaces, since
|
||||
production uses the same output_scale for both."""
|
||||
production uses the same output_scale for both.
|
||||
"""
|
||||
from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
@@ -315,7 +315,8 @@ class TestBackendValidation:
|
||||
|
||||
def test_flash_attn_accepts_glm53_flash_nope_dimensions(self):
|
||||
"""(256, 0, 256) runs the same FA kernels as GLM-5's (192, 64, 256); a
|
||||
RoPE-carrying 320-wide query does not."""
|
||||
RoPE-carrying 320-wide query does not.
|
||||
"""
|
||||
try:
|
||||
from vllm.v1.attention.backends.mla.prefill.flash_attn import (
|
||||
FlashAttnPrefillBackend,
|
||||
|
||||
@@ -257,7 +257,8 @@ def test_replayssm_write_pos(case: ReplaySSMBuildCase):
|
||||
|
||||
def test_resumed_request_differs_from_fresh():
|
||||
"""Same token count, different decode_base: fresh (base 100) -> write_pos 5,
|
||||
resumed (base 105) -> write_pos 0."""
|
||||
resumed (base 105) -> write_pos 0.
|
||||
"""
|
||||
builder = _create_replayssm_builder(16)
|
||||
batch = BatchSpec(seq_lens=[106, 106], query_lens=[1, 1])
|
||||
common = create_common_attn_metadata(batch, BLOCK_SIZE, DEVICE).replace(
|
||||
|
||||
@@ -514,7 +514,6 @@ def test_mtp_builder_init_sizes_native_fp8_metadata(
|
||||
Sweeping num_heads asserts metadata is sized for the padded decode shape,
|
||||
covering Kimi-K3 TP4's 24 -> 32 head path and native fp8 nhead=32 folding.
|
||||
"""
|
||||
|
||||
dtypes = SimpleNamespace(fp8="fp8", fp16="fp16", bf16="bf16")
|
||||
info_calls = []
|
||||
|
||||
@@ -694,7 +693,6 @@ def test_full_cudagraph_padded_uniform_mtp_synthesizes_decode_indptr(
|
||||
monkeypatch,
|
||||
):
|
||||
"""Full-CG zero-qo rows follow rocm_aiter_mla.py:608-657,717-759."""
|
||||
|
||||
get_mla_metadata_v1 = mock.MagicMock()
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
@@ -756,7 +754,6 @@ def test_full_cudagraph_padded_uniform_mtp_synthesizes_decode_indptr(
|
||||
|
||||
def test_decode_expands_kernel_block_page_indices(monkeypatch):
|
||||
"""kernel_block_size>1 expands b -> b*K+offset at rocm_aiter_mla.py:696-704."""
|
||||
|
||||
expand_kernel = _ExpandPageIndicesKernel()
|
||||
monkeypatch.setattr(rocm_aiter_mla, "_expand_page_indices_kernel", expand_kernel)
|
||||
# qlen==1 now takes the persistent-metadata path, which imports
|
||||
|
||||
@@ -517,7 +517,8 @@ def test_auto_selection_for_kv_connector(
|
||||
aiter_found, expected_backend, mock_vllm_config, mock_get_cdna_version
|
||||
):
|
||||
"""Auto-selection with a KV connector and AITER enabled resolves to unified attn,
|
||||
and to triton attn if AITER not enabled."""
|
||||
and to triton attn if AITER not enabled.
|
||||
"""
|
||||
from vllm.platforms.rocm import RocmPlatform
|
||||
|
||||
with patch(
|
||||
|
||||
@@ -102,7 +102,8 @@ def expected_row_seq_lens(per_req_positions, batch_size, next_n):
|
||||
|
||||
def test_legacy_flat_layout_misaligns_and_bleeds():
|
||||
"""The bug: ``positions[:n] + 1`` with padded-row coordinates reads other
|
||||
requests' positions and prefill positions."""
|
||||
requests' positions and prefill positions.
|
||||
"""
|
||||
decode_lens, per_req, positions = make_non_uniform_batch()
|
||||
batch_size = decode_lens.shape[0]
|
||||
next_n = int(decode_lens.max())
|
||||
@@ -137,7 +138,8 @@ def test_helper_uniform_layout_matches_flat_slice():
|
||||
|
||||
def test_helper_padded_layout_per_row():
|
||||
"""Non-uniform batches map every padded row to its own token's position;
|
||||
pad rows get 0 (empty tail)."""
|
||||
pad rows get 0 (empty tail).
|
||||
"""
|
||||
decode_lens, per_req, positions = make_non_uniform_batch()
|
||||
out = _decode_topk_seq_lens(positions, decode_lens, 8, 3, 4, requires_padding=True)
|
||||
expected = expected_row_seq_lens(per_req, 3, 4)
|
||||
@@ -149,7 +151,8 @@ def test_helper_padded_layout_per_row():
|
||||
def expand_tail_region(dec_seq):
|
||||
"""Run the production pure-torch expand + append pair (the fused kernel
|
||||
replicates it exactly on the identity path) and return the tail columns
|
||||
[TOPK_TOKENS, TOPK_TOKENS + KPOOL - 1)."""
|
||||
[TOPK_TOKENS, TOPK_TOKENS + KPOOL - 1).
|
||||
"""
|
||||
rows = dec_seq.shape[0]
|
||||
pool_ids = torch.arange(SELECT_K, dtype=torch.int64).expand(rows, SELECT_K)
|
||||
valid = torch.ones_like(pool_ids, dtype=torch.bool)
|
||||
@@ -163,7 +166,8 @@ def expand_tail_region(dec_seq):
|
||||
def test_tail_expansion_legacy_vs_fixed():
|
||||
"""End-to-end tail consequence: the fixed mapping appends exactly the
|
||||
request's trailing incomplete pool; the legacy flat slice drops real tail
|
||||
tokens and emits indices far past the request's own sequence."""
|
||||
tokens and emits indices far past the request's own sequence.
|
||||
"""
|
||||
decode_lens, per_req, positions = make_non_uniform_batch()
|
||||
n = 12
|
||||
fixed = _decode_topk_seq_lens(
|
||||
|
||||
@@ -245,8 +245,8 @@ def _dequantize_fp8_ds_mla_entry(
|
||||
Args:
|
||||
simulate_sm100_e8m0_scales: If True, simulate the SM100 kernel's
|
||||
float -> e8m0 -> bf16 scale conversion path.
|
||||
"""
|
||||
|
||||
"""
|
||||
# The first kv_lora_rank bytes store FP8 latent values with one scale per
|
||||
# 128 element tile written as float32 right after the latent payload.
|
||||
scales = cache_slice.view(torch.float32)[kv_lora_rank // 4 : kv_lora_rank // 4 + 4]
|
||||
@@ -283,8 +283,8 @@ def _quantize_dequantize_fp8_ds_mla(
|
||||
Args:
|
||||
simulate_sm100_e8m0_scales: If True, simulate the SM100 kernel's
|
||||
float -> e8m0 -> bf16 scale conversion in dequantization.
|
||||
"""
|
||||
|
||||
"""
|
||||
if kv_c.numel() == 0:
|
||||
return kv_c.clone(), k_pe.clone()
|
||||
|
||||
@@ -909,7 +909,8 @@ def test_index_group_convert_during_piecewise_capture():
|
||||
runs in the following eager break. The convert's side stream must not
|
||||
wait on ``logical_topk_ready`` there: an event recorded inside a captured
|
||||
segment is graph-local, and an eager wait on it raises
|
||||
cudaErrorInvalidValue."""
|
||||
cudaErrorInvalidValue.
|
||||
"""
|
||||
device = torch.device(DEVICE_TYPE)
|
||||
num_tokens, num_topk, num_requests, blocks_per_req, block_size = 8, 128, 4, 4, 16
|
||||
|
||||
@@ -1048,7 +1049,8 @@ def test_triton_convert_rejects_req_id_longer_than_token_indices():
|
||||
req_id but the output is allocated like token_indices, so a full-batch
|
||||
req_id combined with an MQA-subset token_indices wrote past the end of
|
||||
the output buffer. The wrapper must reject the length mismatch instead
|
||||
of corrupting memory."""
|
||||
of corrupting memory.
|
||||
"""
|
||||
device = torch.device(DEVICE_TYPE)
|
||||
num_topk_tokens = 128
|
||||
block_size = 64
|
||||
@@ -1103,7 +1105,8 @@ def test_flashmla_forward_bf16_kv_slices_req_id_to_mqa_tokens():
|
||||
is active, forward_mqa only receives the leading decode tokens, but
|
||||
_forward_bf16_kv passed the full-batch req_id_per_token to the index
|
||||
conversion, making it write past the end of its output buffer. The call
|
||||
site must slice req_id_per_token to the MQA tokens."""
|
||||
site must slice req_id_per_token to the MQA tokens.
|
||||
"""
|
||||
device = torch.device(DEVICE_TYPE)
|
||||
num_topk_tokens = 128
|
||||
block_size = 64
|
||||
@@ -1201,7 +1204,8 @@ def test_split_prefill_chunks(seq_lens, max_buf, expected):
|
||||
)
|
||||
def test_masked_mha_workspace_fits_single_request_boundary(max_query_len, expected):
|
||||
"""A 32K prefill needs the default workspace exactly; shrinking it would
|
||||
push a supported request onto MQA."""
|
||||
push a supported request onto MQA.
|
||||
"""
|
||||
assert (
|
||||
_masked_mha_workspace_fits(
|
||||
batch_size=1,
|
||||
@@ -1259,7 +1263,8 @@ def test_is_masked_mha_available_model_dims(
|
||||
):
|
||||
"""The allow-list gates masked MHA per exact model geometry: the DeepSeek-V3.2,
|
||||
GLM-5 and NoPE GLM-5.3-Flash layouts on an SM100-family GPU with FA4 and an
|
||||
unquantized KV cache, nothing else."""
|
||||
unquantized KV cache, nothing else.
|
||||
"""
|
||||
import vllm.model_executor.layers.attention.sparse_mla_attention as mod
|
||||
|
||||
monkeypatch.setattr(
|
||||
@@ -3599,7 +3604,8 @@ def test_fp8_mixed_batch_dcp_neutralizes_empty_rows(monkeypatch):
|
||||
"""A decode row whose top-k shard holds no local candidates (all -1) has
|
||||
undefined kernel out/lse; it must come back as (0, -inf), the identity of
|
||||
the cross-rank LSE merge, or a NaN would survive the merge even at zero
|
||||
weight (0 * NaN = NaN)."""
|
||||
weight (0 * NaN = NaN).
|
||||
"""
|
||||
num_tokens, num_heads, head_dim = 3, 2, 3
|
||||
q = torch.empty(num_tokens, num_heads, head_dim, device=DEVICE_TYPE)
|
||||
local_indices = torch.tensor(
|
||||
@@ -3772,7 +3778,8 @@ def test_hisparse_fp8_prefill_gather_uses_dedicated_stream(monkeypatch):
|
||||
def test_sparse_impl_observes_repointed_indexer_buffer():
|
||||
"""The MTP proposer repoints the draft's indexer at the target model's buffer
|
||||
after the backend impl is built, so the impl must resolve the buffer per read.
|
||||
Snapshotting it in __init__ leaves the layer reading indices nothing writes."""
|
||||
Snapshotting it in __init__ leaves the layer reading indices nothing writes.
|
||||
"""
|
||||
impl = object.__new__(FlashInferMLASparseImpl)
|
||||
own = torch.zeros(4, 8, dtype=torch.int32)
|
||||
target = torch.ones(4, 8, dtype=torch.int32)
|
||||
@@ -3788,7 +3795,8 @@ def test_sparse_impl_observes_repointed_indexer_buffer():
|
||||
|
||||
def test_explicit_topk_buffer_supersedes_indexer():
|
||||
"""Backbone skip-topk layers have no indexer, and the proposer also assigns the
|
||||
shared buffer directly onto draft submodules."""
|
||||
shared buffer directly onto draft submodules.
|
||||
"""
|
||||
impl = object.__new__(FlashInferMLASparseImpl)
|
||||
indexer = SimpleNamespace(
|
||||
topk_indices_buffer=torch.zeros(2, 2, dtype=torch.int32),
|
||||
@@ -3803,7 +3811,8 @@ def test_explicit_topk_buffer_supersedes_indexer():
|
||||
|
||||
def test_sparse_mla_common_impl_resolves_buffer_lazily():
|
||||
"""Guards the whole SparseMLACommonImpl family at once: a backend that
|
||||
snapshots the buffer instead silently loses MTP buffer sharing."""
|
||||
snapshots the buffer instead silently loses MTP buffer sharing.
|
||||
"""
|
||||
assert issubclass(SparseMLACommonImpl, SharedTopkIndicesBuffer)
|
||||
assert isinstance(SparseMLACommonImpl.topk_indices_buffer, property), (
|
||||
"topk_indices_buffer must stay a lazily-resolved property"
|
||||
|
||||
@@ -124,6 +124,7 @@ def _create_nvfp4_hnd_kv_cache(
|
||||
|
||||
Returns:
|
||||
``torch.Tensor``: The nvfp4 kv_cache tensor (uint8, LBHNC-strided).
|
||||
|
||||
"""
|
||||
# First create a bf16 cache so block tables are populated.
|
||||
bf16_cache = create_and_prepopulate_kv_cache(
|
||||
@@ -197,7 +198,8 @@ def _create_nvfp4_hnd_kv_cache(
|
||||
|
||||
def _run_trtllm_integration(batch_spec, kv_cache_dtype="auto", model_name=MODEL):
|
||||
"""Run TRTLLM attention through the full FlashInfer pipeline
|
||||
and compare against an SDPA reference."""
|
||||
and compare against an SDPA reference.
|
||||
"""
|
||||
set_random_seed(42)
|
||||
device = torch.device(f"{DEVICE_TYPE}:0")
|
||||
|
||||
@@ -425,7 +427,8 @@ def _run_trtllm_integration(batch_spec, kv_cache_dtype="auto", model_name=MODEL)
|
||||
def test_trtllm_gen_full_attention_integration(batch_spec_name: str):
|
||||
"""Test TRTLLM gen-full attention through the full FlashInfer
|
||||
MetadataBuilder.build() -> FlashInferImpl.forward() pipeline,
|
||||
with real TRTLLM kernels on Blackwell."""
|
||||
with real TRTLLM kernels on Blackwell.
|
||||
"""
|
||||
_run_trtllm_integration(BATCH_SPECS[batch_spec_name])
|
||||
|
||||
|
||||
@@ -436,7 +439,8 @@ def test_trtllm_gen_full_attention_integration(batch_spec_name: str):
|
||||
@torch.inference_mode()
|
||||
def test_trtllm_gen_nvfp4_kv_integration(batch_spec_name: str):
|
||||
"""Test TRTLLM attention with nvfp4 KV cache through the full
|
||||
FlashInfer MetadataBuilder.build() -> FlashInferImpl.forward() pipeline."""
|
||||
FlashInfer MetadataBuilder.build() -> FlashInferImpl.forward() pipeline.
|
||||
"""
|
||||
_run_trtllm_integration(
|
||||
BATCH_SPECS[batch_spec_name],
|
||||
kv_cache_dtype="nvfp4",
|
||||
|
||||
@@ -193,7 +193,6 @@ def create_vllm_config(
|
||||
hf_config_override: dict | None = None,
|
||||
) -> VllmConfig:
|
||||
"""Create a VllmConfig for testing with reasonable defaults."""
|
||||
|
||||
model_config = ModelConfig(
|
||||
model=model_name,
|
||||
tokenizer=model_name,
|
||||
|
||||
@@ -2,7 +2,8 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Fine-grained partial prefix-cache hits for hybrid (full attention + mamba
|
||||
"align") models: scheduler chunk splitting, partial tail registration, CoW
|
||||
on partial hits, and same-step deferral."""
|
||||
on partial hits, and same-step deferral.
|
||||
"""
|
||||
|
||||
from math import lcm
|
||||
from types import SimpleNamespace
|
||||
@@ -174,7 +175,8 @@ def test_mamba_align_split_partial_tail_schedule(dcp_world_size: int):
|
||||
"""Chunk ends with partial hits on: block-aligned chunks, one extra stop
|
||||
at the prompt's last hash boundary (registering the partial tail), then
|
||||
the remaining tokens. block=512, hash=32, prompt=10000, budget=8192:
|
||||
0 -> 8192 -> 9728 -> 9984 -> 10000."""
|
||||
0 -> 8192 -> 9728 -> 9984 -> 10000.
|
||||
"""
|
||||
block_size = 512
|
||||
scheduler_block_size = block_size * dcp_world_size
|
||||
hash_block_size = 32
|
||||
@@ -742,7 +744,8 @@ def test_hash_aligned_query_end_uses_regular_partial_tail():
|
||||
|
||||
def test_external_mamba_hit_same_block_uses_running_cow_on_continue():
|
||||
"""An external mid-block hit must become a running request even when its
|
||||
first continuation does not need another Mamba block."""
|
||||
first continuation does not need another Mamba block.
|
||||
"""
|
||||
hash_block_size = 2
|
||||
mamba_block_size = 4 * hash_block_size
|
||||
kv_cache_config = KVCacheConfig(
|
||||
@@ -814,7 +817,8 @@ def test_external_mamba_hit_same_block_uses_running_cow_on_continue():
|
||||
|
||||
def test_boundary_state_offloads_returns_cow_target():
|
||||
"""Boundary hand-offs expose aligned snapshots and the partial-tail CoW
|
||||
target, never the overwritten CoW source."""
|
||||
target, never the overwritten CoW source.
|
||||
"""
|
||||
hash_block_size = 2
|
||||
block_size = 2 * hash_block_size
|
||||
kv_cache_config = KVCacheConfig(
|
||||
@@ -1000,7 +1004,8 @@ def test_connector_finish_registers_partial_tail_before_cleanup():
|
||||
def test_boundary_state_offload_dropped_when_request_freed_before_drain():
|
||||
"""A hand-off recorded in the same scheduling pass as the request's death
|
||||
must not be drained: its release hook has already run, so draining would
|
||||
leak a pinned block."""
|
||||
leak a pinned block.
|
||||
"""
|
||||
hash_block_size = 2
|
||||
block_size = 2 * hash_block_size
|
||||
kv_cache_config = KVCacheConfig(
|
||||
@@ -1047,7 +1052,8 @@ def test_boundary_state_offload_dropped_when_request_freed_before_drain():
|
||||
|
||||
def test_boundary_state_offloads_block_aligned_prompt():
|
||||
"""A prompt ending on a block boundary registers no CoW partial tail; its
|
||||
boundary state block is handed off as a snapshot instead (once)."""
|
||||
boundary state block is handed off as a snapshot instead (once).
|
||||
"""
|
||||
hash_block_size = 2
|
||||
block_size = 2 * hash_block_size
|
||||
kv_cache_config = KVCacheConfig(
|
||||
@@ -1101,7 +1107,8 @@ def test_boundary_state_offloads_block_aligned_prompt():
|
||||
def test_truncate_computed_blocks_preserves_sparse_prefix_positions():
|
||||
"""truncate_computed_blocks slices each group by its own block size,
|
||||
keeps null placeholders in the retained prefix, and leaves the original
|
||||
lookup result untouched (pure view, no refcount changes)."""
|
||||
lookup result untouched (pure view, no refcount changes).
|
||||
"""
|
||||
hash_block_size = 2
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=24,
|
||||
@@ -1156,7 +1163,8 @@ def test_truncate_computed_blocks_preserves_sparse_prefix_positions():
|
||||
|
||||
def test_truncate_computed_blocks_allows_short_mamba_group_only():
|
||||
"""External state may replace a short Mamba hit, but other groups must
|
||||
cover the aligned local endpoint."""
|
||||
cover the aligned local endpoint.
|
||||
"""
|
||||
hash_block_size = 2
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=24,
|
||||
@@ -1298,7 +1306,8 @@ def test_hybrid_mamba_partial_tail_owner_continue_preserves_later_hit():
|
||||
def test_hybrid_mamba_moved_partial_entry_defers_same_step_hit():
|
||||
"""The owner's move re-arms the same-step guard: the moved entry is
|
||||
filled by this step's copy, and chained same-step copies read stale
|
||||
sources, so a request hitting it in the move step must be deferred."""
|
||||
sources, so a request hitting it in the move step must be deferred.
|
||||
"""
|
||||
hash_block_size = 2
|
||||
block_size = 2 * hash_block_size
|
||||
kv_cache_config = KVCacheConfig(
|
||||
@@ -1588,7 +1597,8 @@ def test_hybrid_partial_hash_truncates_full_attention_hit_length():
|
||||
|
||||
def test_cow_retained_blocks_returned_for_release():
|
||||
"""new_step_starts returns the CoW copy retentions instead of freeing
|
||||
them; the scheduler owns releasing them once the copy has run."""
|
||||
them; the scheduler owns releasing them once the copy has run.
|
||||
"""
|
||||
hash_block_size = 2
|
||||
block_size = 2 * hash_block_size
|
||||
kv_cache_config = KVCacheConfig(
|
||||
@@ -1641,7 +1651,8 @@ def test_cow_retained_blocks_returned_for_release():
|
||||
|
||||
def test_free_cow_retained_blocks_defers_until_copy_step_processed():
|
||||
"""Scheduler releases CoW retentions immediately when the copy's step has
|
||||
been processed (or deferral is off), and defers them otherwise."""
|
||||
been processed (or deferral is off), and defers them otherwise.
|
||||
"""
|
||||
from collections import deque
|
||||
|
||||
freed: list = []
|
||||
@@ -1678,7 +1689,8 @@ def test_free_cow_retained_blocks_defers_until_copy_step_processed():
|
||||
def test_full_attention_eagle_drops_one_hash_unit():
|
||||
"""With fine-grained partial hits, eagle rewinds the hit by one hash unit
|
||||
instead of a whole cache block: the tail block's KV is append-only, so it
|
||||
still covers the reduced length and stays in the hit as a partial block."""
|
||||
still covers the reduced length and stays in the hit as a partial block.
|
||||
"""
|
||||
from vllm.v1.core.block_pool import BlockPool
|
||||
from vllm.v1.core.single_type_kv_cache_manager import FullAttentionManager
|
||||
|
||||
@@ -1754,7 +1766,8 @@ def test_hybrid_partial_hit_with_eagle_stays_within_group_blocks():
|
||||
"""Regression: with eagle, the mamba group must not receive the eagle
|
||||
lookup margin — its finder never applies the drop, so it could return a
|
||||
hit past the blocks the (dropped) full-attention group covers, crashing
|
||||
the consumer's CoW with block_idx >= len(req_blocks)."""
|
||||
the consumer's CoW with block_idx >= len(req_blocks).
|
||||
"""
|
||||
hash_block_size = 2
|
||||
block_size = 2 * hash_block_size
|
||||
kv_cache_config = KVCacheConfig(
|
||||
@@ -2356,7 +2369,8 @@ def test_retention_snapshots_handed_off_with_exact_block_ids():
|
||||
"""Under sparse retention, each retained mamba boundary state block is
|
||||
handed to the connector with its exact block id. Connectors must not resolve
|
||||
align-mode state blocks positionally because the block table is not
|
||||
append-only."""
|
||||
append-only.
|
||||
"""
|
||||
hash_block_size = 2
|
||||
block_size = 4
|
||||
manager = make_kv_cache_manager(
|
||||
@@ -2397,7 +2411,8 @@ def test_snapshot_handoff_dense_default_retention():
|
||||
"""With dense (default) retention, every materialized mamba boundary
|
||||
state block is handed off — regular mamba-align + prefix-match-unit
|
||||
deployments get store-able boundary snapshots without setting a
|
||||
prefix-cache retention interval."""
|
||||
prefix-cache retention interval.
|
||||
"""
|
||||
hash_block_size = 2
|
||||
block_size = 4
|
||||
manager = make_kv_cache_manager(
|
||||
@@ -2429,7 +2444,8 @@ def test_snapshot_handoff_dense_default_retention():
|
||||
|
||||
def _run_chunked_prefill(manager, req, chunk_size, num_chunks):
|
||||
"""Prefill ``req`` one ``chunk_size`` chunk per scheduler step, yielding the
|
||||
boundary-state hand-offs offered (and claimed) at each step."""
|
||||
boundary-state hand-offs offered (and claimed) at each step.
|
||||
"""
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req)
|
||||
for step in range(num_chunks):
|
||||
manager.new_step_starts()
|
||||
@@ -2493,7 +2509,8 @@ def test_boundary_states_offered_past_prompt_for_resumed_prefill():
|
||||
``num_prompt_tokens``. A resumed request re-prefills its generated tokens
|
||||
and every group re-saves them, so filtering on the original prompt length
|
||||
would silently strip the mamba key for boundaries full attention still
|
||||
stores; only the connector knows where its save window ends."""
|
||||
stores; only the connector knows where its save window ends.
|
||||
"""
|
||||
block_size = 8
|
||||
prompt_len = block_size
|
||||
manager = make_kv_cache_manager(
|
||||
|
||||
@@ -552,7 +552,8 @@ def _create_async_pp_scheduler(
|
||||
|
||||
def _assert_ordered_subset(delivered: list[int], emitted: list[int]) -> None:
|
||||
"""Delivered tokens must be an order-preserving subset of the emitted
|
||||
tokens with no duplicates (tokens are globally unique)."""
|
||||
tokens with no duplicates (tokens are globally unique).
|
||||
"""
|
||||
it = iter(emitted)
|
||||
for token in delivered:
|
||||
assert token in it, f"token {token} delivered out of order or twice"
|
||||
@@ -562,7 +563,8 @@ def _assert_positions_consistent(req, engine: PipelinedEngine) -> None:
|
||||
"""The i-th delivered output token must be one the runner sampled for
|
||||
exactly sequence position prompt_len + i: catches a preempted request's
|
||||
stale output landing on a position the resumed request resampled (or
|
||||
vice versa), which token-stream equality alone cannot see."""
|
||||
vice versa), which token-stream equality alone cannot see.
|
||||
"""
|
||||
for i, token in enumerate(req.output_token_ids):
|
||||
expected = req.num_prompt_tokens + i
|
||||
actual = engine.emitted_position[token]
|
||||
|
||||
@@ -178,7 +178,8 @@ def _shared_layout_config():
|
||||
class TestCSALinearGrouping:
|
||||
"""A CSA + linear-attention model (sparse attention with a compressor ring,
|
||||
plus sharded GDN and one TP-replicated PLE state) goes through the generic
|
||||
packed-group path; no model-specific branch is involved."""
|
||||
packed-group path; no model-specific branch is involved.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _mamba_groups(groups):
|
||||
@@ -193,7 +194,8 @@ class TestCSALinearGrouping:
|
||||
def test_replicated_state_gets_its_own_group(self):
|
||||
"""The PLE state is TP-replicated and a different size, so it must not
|
||||
share a manager group with the sharded GDN states (the NIXL worker
|
||||
requires exactly one single-layer replicated group)."""
|
||||
requires exactly one single-layer replicated group).
|
||||
"""
|
||||
groups = get_kv_cache_groups(_shared_layout_config(), _make_csa_linear_specs())
|
||||
|
||||
mamba_groups = self._mamba_groups(groups)
|
||||
@@ -246,7 +248,8 @@ class TestCSALinearGrouping:
|
||||
def test_scratch_group_survives_computed_block_truncation(self):
|
||||
"""The scratch group contributes no computed blocks, so truncating a
|
||||
lookup result must skip it: its block size is the ring capacity, which
|
||||
neither divides the hit length nor bounds the (empty) block list."""
|
||||
neither divides the hit length nor bounds the (empty) block list.
|
||||
"""
|
||||
config = _shared_layout_config()
|
||||
config.cache_config.enable_prefix_caching = True
|
||||
groups = get_kv_cache_groups(config, _make_csa_linear_specs())
|
||||
@@ -360,7 +363,8 @@ class TestCSALinearGrouping:
|
||||
per page size, which is emitted whole, or a single-layer state that
|
||||
cannot be split at all -- the mamba layers must be sized against it.
|
||||
Sizing them against a narrower bucket splits them past what the block
|
||||
already fits, spending a pool block per extra group for no saving."""
|
||||
already fits, spending a pool block per extra group for no saving.
|
||||
"""
|
||||
config = _mock_vllm_config("BLNHC")
|
||||
config.speculative_config = None
|
||||
specs = {}
|
||||
@@ -405,7 +409,8 @@ class TestSlidingWindowBucketCap:
|
||||
unbalanced 8-layer paged MLA bucket. Left whole, the SWA bucket would
|
||||
set the block width and the MLA group would fill under a third of
|
||||
every block; capped, it splits into groups no wider than the MLA
|
||||
page."""
|
||||
page.
|
||||
"""
|
||||
config = _shared_layout_config()
|
||||
specs: dict[str, KVCacheSpec] = {}
|
||||
for layer, ratio in ((2, 2), (8, 2), (14, 2), (20, 1)):
|
||||
@@ -606,7 +611,8 @@ class TestCompressorRingGroup:
|
||||
"""DeepSeek-V4.1 compressor state: a CircularBufferSpec ring (capacity
|
||||
8) beside 128-token paged MLA groups and 64-token SWA groups. The ring
|
||||
claims one block per request, never hashes, and does not disturb the
|
||||
scheduler/hash block sizes the paged groups imply."""
|
||||
scheduler/hash block sizes the paged groups imply.
|
||||
"""
|
||||
config = _shared_layout_config()
|
||||
config.cache_config.enable_prefix_caching = True
|
||||
specs: dict[str, KVCacheSpec] = {
|
||||
|
||||
@@ -367,7 +367,8 @@ def test_free_request_with_duplicate_mm_hashes():
|
||||
"""Freeing a request whose two inputs share the same mm_hash must fully
|
||||
clean up request_cached_ids. After the first free_encoder_input call,
|
||||
cached[mm_hash] becomes empty; the second call must still remove the
|
||||
remaining input_id from request_cached_ids."""
|
||||
remaining input_id from request_cached_ids.
|
||||
"""
|
||||
manager = EncoderCacheManager(cache_size=20)
|
||||
|
||||
req = MockRequest("r1", ["imgA", "imgA"], [4, 4])
|
||||
@@ -383,7 +384,8 @@ def test_free_request_with_duplicate_mm_hashes():
|
||||
|
||||
def test_duplicate_mm_hash_stays_referenced_until_last_free():
|
||||
"""`cached` holds one reference per request, not per position, so freeing
|
||||
the first of two occurrences must not release the entry."""
|
||||
the first of two occurrences must not release the entry.
|
||||
"""
|
||||
manager = EncoderCacheManager(cache_size=20)
|
||||
req = MockRequest("r1", ["imgA", "imgA"], [4, 4])
|
||||
|
||||
@@ -404,7 +406,8 @@ def test_duplicate_mm_hash_stays_referenced_until_last_free():
|
||||
def test_duplicate_mm_hash_is_not_evicted_before_last_use():
|
||||
"""An item repeated within one request (an image carried across
|
||||
conversation turns) must not become reclaimable capacity while a later
|
||||
occurrence still has to be spliced in, or the encoder recomputes it."""
|
||||
occurrence still has to be spliced in, or the encoder recomputes it.
|
||||
"""
|
||||
manager = EncoderCacheManager(cache_size=8)
|
||||
req = MockRequest("r1", ["imgA", "imgA"], [4, 4])
|
||||
other = MockRequest("r2", ["imgB"], [8])
|
||||
|
||||
@@ -1069,9 +1069,7 @@ def _stats(requests: int, queries: int, hits: int) -> PrefixCacheStats:
|
||||
|
||||
|
||||
def test_metrics():
|
||||
"""
|
||||
Test the prefix caching metrics.
|
||||
"""
|
||||
"""Test the prefix caching metrics."""
|
||||
metrics = CachingMetrics(max_recent_requests=5)
|
||||
assert metrics.hit_rate == 0.0
|
||||
|
||||
@@ -1101,9 +1099,7 @@ def test_metrics():
|
||||
|
||||
|
||||
def test_metrics_empty_stats():
|
||||
"""
|
||||
Test the prefix caching metrics with empty stats.
|
||||
"""
|
||||
"""Test the prefix caching metrics with empty stats."""
|
||||
metrics = CachingMetrics(max_recent_requests=5)
|
||||
metrics.observe(_stats(0, 0, 0))
|
||||
metrics.observe(_stats(1, 20, 9))
|
||||
@@ -1903,7 +1899,7 @@ def test_get_max_concurrency_for_kv_cache_config():
|
||||
|
||||
|
||||
def test_allocate_with_lookahead():
|
||||
"""Verify that lookahead tokens correctly affect block allocation"""
|
||||
"""Verify that lookahead tokens correctly affect block allocation."""
|
||||
block_size = 4
|
||||
config = KVCacheConfig(
|
||||
num_blocks=10,
|
||||
@@ -2597,7 +2593,8 @@ def test_get_kv_cache_config_kpool_tail_coowns_indexer_tensor():
|
||||
"""The kpool tail parasitizes the indexer tensors instead of getting its
|
||||
own: sibling idx/tail tensors paired by layer order, zero standalone tail
|
||||
bytes, one shared block per request, and no prefix-caching leakage from
|
||||
the kpool-sized scratch group."""
|
||||
the kpool-sized scratch group.
|
||||
"""
|
||||
model_config = ModelConfig(max_model_len=8192)
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
|
||||
@@ -2668,7 +2665,8 @@ def test_get_kv_cache_config_kpool_tail_coowns_indexer_tensor():
|
||||
|
||||
def test_glm5_kpool_tail_does_not_drag_hash_block_size():
|
||||
"""The tail's kpool-sized scratch block (4 tokens) must not constrain the
|
||||
prefix-cache hash granularity: participating groups alone decide it."""
|
||||
prefix-cache hash granularity: participating groups alone decide it.
|
||||
"""
|
||||
model_config = ModelConfig(max_model_len=8192)
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
|
||||
@@ -2744,7 +2742,8 @@ def test_get_kv_cache_config_mamba_hybrid_sharing_infeasible_no_indexer():
|
||||
|
||||
def test_get_kv_cache_config_mamba_hybrid_sharing_prepadded_mamba():
|
||||
"""Platform-prepadded mamba pages (mamba_page_size_padded hint) must not
|
||||
disable slot sharing; the layout re-pads them to the MLA page."""
|
||||
disable slot sharing; the layout re-pads them to the MLA page.
|
||||
"""
|
||||
model_config = ModelConfig(max_model_len=8192)
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
|
||||
@@ -2775,7 +2774,8 @@ def test_get_kv_cache_config_mamba_hybrid_sharing_prepadded_mamba():
|
||||
def test_get_kv_cache_config_mamba_hybrid_sharing_pp_balanced_projection():
|
||||
"""Round-robin mamba grouping keeps practical PP splits balanced: every
|
||||
stage's largest projected mamba group slice fits its projected MLA
|
||||
layers, so per-stage slot tensors all have an MLA owner."""
|
||||
layers, so per-stage slot tensors all have an MLA owner.
|
||||
"""
|
||||
model_config = ModelConfig(max_model_len=8192)
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
|
||||
@@ -2819,7 +2819,8 @@ def test_get_kv_cache_config_mamba_hybrid_sharing_pp_group_count_bump(monkeypatc
|
||||
"""PP=4's default partition [11,11,12,11] leaves stage 0 with 2 MLA
|
||||
layers but a round-robin slice of 3 under the minimum 4 mamba groups;
|
||||
grouping bumps to 5 groups so every stage's projection keeps sharing
|
||||
on instead of silently falling back on stage 0."""
|
||||
on instead of silently falling back on stage 0.
|
||||
"""
|
||||
from vllm.config import ParallelConfig
|
||||
from vllm.distributed.utils import get_pp_indices
|
||||
|
||||
@@ -2868,7 +2869,8 @@ def test_get_kv_cache_config_mamba_hybrid_sharing_pp_starved_stage(monkeypatch):
|
||||
def test_get_kv_cache_config_mamba_hybrid_sharing_beats_cross_layers_flag():
|
||||
"""Hybrid slot sharing must take precedence over the experimental
|
||||
enable_cross_layers_blocks packed layout: generic packing would give the
|
||||
MLA slots a strided view, breaking the contiguous virtual split."""
|
||||
MLA slots a strided view, breaking the contiguous virtual split.
|
||||
"""
|
||||
model_config = ModelConfig(max_model_len=8192)
|
||||
kv_transfer_config = KVTransferConfig(
|
||||
kv_connector="NixlConnector",
|
||||
@@ -3170,7 +3172,8 @@ def test_multi_run_layer_compact_strides_place_hoisted_heads():
|
||||
"""A layer-compact run region is its own dense allocation: under LHBNC the head
|
||||
groups sit between the layers and the blocks, so a run's block stride is one head
|
||||
group's slice, not the whole page (regression test for setStorage out-of-bounds
|
||||
on ROCm hybrid models)."""
|
||||
on ROCm hybrid models).
|
||||
"""
|
||||
full = new_kv_cache_spec()
|
||||
swa = new_sliding_window_spec(sliding_window=full.block_size * 2)
|
||||
page = full.page_size_bytes
|
||||
@@ -3249,7 +3252,8 @@ def test_mla_with_incompatible_swa_uses_one_full_allocation_group(caplog_vllm):
|
||||
def test_hidden_states_with_tp_scales_page_size():
|
||||
"""When TP shrinks KV pages below the hidden-state per-token cost,
|
||||
get_kv_cache_groups must scale up target block sizes so that the
|
||||
common page accommodates the unsharded hidden states."""
|
||||
common page accommodates the unsharded hidden states.
|
||||
"""
|
||||
# Simulate TP=4 sharding a model with 8 KV heads → 2 per rank.
|
||||
# KV page = block_size(16) * num_kv_heads(2) * head_size(64) * dtype(2)
|
||||
# = 16 * 2 * 64 * 2 = 4096 bytes.
|
||||
@@ -3922,8 +3926,7 @@ def test_unify_kv_cache_spec_page_size_mamba():
|
||||
|
||||
|
||||
def test_hma_not_disabled_when_kv_events_enabled():
|
||||
"""
|
||||
Test enabling KV events must not force disable_hybrid_kv_cache_manager to True.
|
||||
"""Test enabling KV events must not force disable_hybrid_kv_cache_manager to True.
|
||||
|
||||
This test guards against that regression by verifying that a VllmConfig
|
||||
with kv_events_config set still resolves disable_hybrid_kv_cache_manager
|
||||
@@ -4080,7 +4083,8 @@ def test_check_enough_kv_cache_memory_reserves_null_block():
|
||||
|
||||
def test_is_full_attention_spec_unwraps_uniform_type_specs():
|
||||
"""``UniformTypeKVCacheSpecs`` is not a ``FullAttentionSpec``, so callers
|
||||
scanning groups with a bare isinstance miss DeepSeek-V4-shaped configs."""
|
||||
scanning groups with a bare isinstance miss DeepSeek-V4-shaped configs.
|
||||
"""
|
||||
common = dict(num_kv_heads=1, head_size=1, dtype=torch.float32)
|
||||
full = FullAttentionSpec(block_size=4, **common)
|
||||
mla = MLAAttentionSpec(block_size=4, **common)
|
||||
@@ -4249,7 +4253,8 @@ def test_no_warning_when_draft_group_is_identified(caplog_vllm):
|
||||
|
||||
def _deepseek_v4_specs(model_version="deepseek_v4"):
|
||||
"""DeepseekV4-shaped specs: full MLA layers plus sliding-window MLA layers
|
||||
at differing page sizes, with the MTP draft layer registered last."""
|
||||
at differing page sizes, with the MTP draft layer registered last.
|
||||
"""
|
||||
return {
|
||||
"model.layers.0.self_attn.attn": new_mla_spec(),
|
||||
"model.layers.1.self_attn.attn": new_mla_spec(),
|
||||
|
||||
@@ -17,8 +17,7 @@ def new_kv_cache_spec():
|
||||
|
||||
|
||||
def test_initialize_kv_cache_for_kv_sharing_different_attn_groups():
|
||||
"""
|
||||
Test initializing KV cache sharing with different attention groups.
|
||||
"""Test initializing KV cache sharing with different attention groups.
|
||||
Layers in the same KV cache group might be placed in different attn groups
|
||||
if they have different attention backends.
|
||||
"""
|
||||
@@ -50,8 +49,7 @@ def test_initialize_kv_cache_for_kv_sharing_different_attn_groups():
|
||||
|
||||
|
||||
def test_initialize_kv_cache_for_kv_sharing_same_attn_groups():
|
||||
"""
|
||||
Test case assuming that all layers in the same KV cache group have the same
|
||||
"""Test case assuming that all layers in the same KV cache group have the same
|
||||
attention backends. This is true for most models.
|
||||
"""
|
||||
shared_kv_cache_layers = {
|
||||
@@ -79,8 +77,7 @@ def test_initialize_kv_cache_for_kv_sharing_same_attn_groups():
|
||||
|
||||
|
||||
def test_initialize_kv_cache_for_kv_sharing_no_attn_groups():
|
||||
"""
|
||||
Test KV sharing set up when no attention groups are provided.
|
||||
"""Test KV sharing set up when no attention groups are provided.
|
||||
This is the case for the TPU model runner, which doesn't have
|
||||
support for attention groups yet.
|
||||
"""
|
||||
|
||||
@@ -55,7 +55,8 @@ def _mm_feature(offset: int, length: int) -> MultiModalFeatureSpec:
|
||||
|
||||
def test_strip_covered_mm_data() -> None:
|
||||
"""Items fully inside the computed prefix lose their data; items touching
|
||||
the uncomputed region keep it; already-None data stays None."""
|
||||
the uncomputed region keep it; already-None data stays None.
|
||||
"""
|
||||
from dataclasses import replace
|
||||
|
||||
covered = _mm_feature(offset=0, length=100)
|
||||
@@ -107,7 +108,8 @@ def _mm_feature_mixed(offset: int, length: int) -> MultiModalFeatureSpec:
|
||||
|
||||
def test_strip_covered_mm_data_mrope() -> None:
|
||||
"""For M-RoPE models, covered items keep their keep_on_cpu metadata fields
|
||||
(the worker needs them to compute positions); payload fields are dropped."""
|
||||
(the worker needs them to compute positions); payload fields are dropped.
|
||||
"""
|
||||
covered = _mm_feature_mixed(offset=0, length=100)
|
||||
uncovered = _mm_feature_mixed(offset=300, length=100)
|
||||
|
||||
@@ -127,7 +129,8 @@ def test_strip_covered_mm_data_shm_address_item() -> None:
|
||||
"""SHM address descriptors must survive stripping even when covered: the
|
||||
worker needs the address to resolve the payload and to acknowledge the
|
||||
sender's writer reference. Stripping it crashed the SHM receiver cache on
|
||||
the second identical request (vllm-project/vllm#54994)."""
|
||||
the second identical request (vllm-project/vllm#54994).
|
||||
"""
|
||||
address_item = MultiModalKwargsItem(
|
||||
{
|
||||
"address": MultiModalFieldElem(data=4096, field=MultiModalBatchedField()),
|
||||
|
||||
@@ -126,7 +126,8 @@ def make_kv_cache_manager(kv_cache_config: KVCacheConfig, **kwargs) -> KVCacheMa
|
||||
"""Build a ``KVCacheManager``, deriving ``scheduler_block_size`` from the
|
||||
config (LCM of group block sizes) unless explicitly provided. This mirrors
|
||||
``resolve_kv_cache_block_sizes`` for the non-context-parallel case used by
|
||||
these tests, so callers don't have to pass it at every site."""
|
||||
these tests, so callers don't have to pass it at every site.
|
||||
"""
|
||||
kwargs.setdefault(
|
||||
"scheduler_block_size",
|
||||
lcm(*(g.kv_cache_spec.block_size for g in kv_cache_config.kv_cache_groups)),
|
||||
@@ -1826,8 +1827,7 @@ def _test_partial_request_hit(
|
||||
def _make_hybrid_kv_cache_config(
|
||||
block_size: int, num_blocks: int, spec_types: list[str]
|
||||
) -> KVCacheConfig:
|
||||
"""
|
||||
Create a KVCacheConfig with the specified spec types.
|
||||
"""Create a KVCacheConfig with the specified spec types.
|
||||
|
||||
Args:
|
||||
block_size: The block size for KV cache.
|
||||
@@ -1837,6 +1837,7 @@ def _make_hybrid_kv_cache_config(
|
||||
- "sliding_window": SlidingWindowSpec with window=2*block_size
|
||||
- "sliding_window_large": SlidingWindowSpec with window=4*block_size
|
||||
- "mamba": MambaSpec
|
||||
|
||||
"""
|
||||
spec_map = {
|
||||
"full": lambda: FullAttentionSpec(
|
||||
@@ -1938,8 +1939,7 @@ _HYBRID_MODEL_TEST_CASES = [
|
||||
|
||||
@pytest.mark.parametrize("spec_types", _HYBRID_MODEL_TEST_CASES)
|
||||
def test_prefill_hybrid_model_combinations(spec_types: list[str]):
|
||||
"""
|
||||
Test prefix caching with hybrid models containing various combinations of
|
||||
"""Test prefix caching with hybrid models containing various combinations of
|
||||
KV cache spec types.
|
||||
|
||||
This unified test covers:
|
||||
@@ -2021,8 +2021,7 @@ _EAGLE_HYBRID_MODEL_TEST_CASES = [
|
||||
def test_prefill_hybrid_model_combinations_eagle(
|
||||
spec_types: list[str], expect_hit_length: int
|
||||
):
|
||||
"""
|
||||
Test prefix caching with hybrid models (1 full attn + 1 other) with EAGLE.
|
||||
"""Test prefix caching with hybrid models (1 full attn + 1 other) with EAGLE.
|
||||
More complex hybrid models with EAGLE are not yet supported (see issue #32802).
|
||||
"""
|
||||
block_size = 16
|
||||
@@ -2489,8 +2488,7 @@ def test_evict():
|
||||
|
||||
|
||||
def test_hash_block_correct_reuse():
|
||||
"""
|
||||
This tests when a previously cached block is reused as a new block,
|
||||
"""This tests when a previously cached block is reused as a new block,
|
||||
its hash metadata should be correctly reset.
|
||||
"""
|
||||
block_size = 16
|
||||
@@ -2530,8 +2528,7 @@ def test_hash_block_correct_reuse():
|
||||
|
||||
|
||||
def test_computed_blocks_not_evicted():
|
||||
"""
|
||||
Test that the computed blocks are not evicted when getting new blocks
|
||||
"""Test that the computed blocks are not evicted when getting new blocks
|
||||
for a request if there are any other free blocks.
|
||||
"""
|
||||
block_size = 16
|
||||
@@ -2590,9 +2587,7 @@ def test_computed_blocks_not_evicted():
|
||||
|
||||
|
||||
def test_basic_prefix_caching_disabled():
|
||||
"""
|
||||
This tests that the prefix caching is disabled.
|
||||
"""
|
||||
"""This tests that the prefix caching is disabled."""
|
||||
block_size = 4
|
||||
manager = make_kv_cache_manager(
|
||||
make_kv_cache_config(block_size, 5),
|
||||
@@ -2639,11 +2634,9 @@ def test_basic_prefix_caching_disabled():
|
||||
|
||||
@pytest.mark.parametrize("hash_fn", [sha256, sha256_cbor])
|
||||
def test_cache_blocks(hash_fn):
|
||||
"""
|
||||
This is a unit test that tests the correctness of the _cache_full_blocks
|
||||
"""This is a unit test that tests the correctness of the _cache_full_blocks
|
||||
function of KVCacheManager.
|
||||
"""
|
||||
|
||||
block_size = 4
|
||||
block_pool = BlockPool(
|
||||
num_gpu_blocks=5,
|
||||
@@ -2688,9 +2681,7 @@ def test_cache_blocks(hash_fn):
|
||||
|
||||
|
||||
def test_cache_blocks_multi_group():
|
||||
"""
|
||||
This tests that blocks are cached correctly for different kv cache groups.
|
||||
"""
|
||||
"""This tests that blocks are cached correctly for different kv cache groups."""
|
||||
block_size = 4
|
||||
block_pool = BlockPool(
|
||||
num_gpu_blocks=10, enable_caching=True, hash_block_size=block_size
|
||||
@@ -2773,10 +2764,7 @@ def test_cache_blocks_multi_group():
|
||||
|
||||
|
||||
def test_mm_prefix_caching():
|
||||
"""
|
||||
This tests that the multi-modal prefix caching is correct.
|
||||
"""
|
||||
|
||||
"""This tests that the multi-modal prefix caching is correct."""
|
||||
block_size = 16
|
||||
manager = make_kv_cache_manager(
|
||||
make_kv_cache_config(block_size, 11),
|
||||
@@ -2881,8 +2869,7 @@ def test_mm_prefix_caching():
|
||||
|
||||
|
||||
def test_cache_key_salting():
|
||||
"""
|
||||
This tests that cache salts are applied during hashing and the cache
|
||||
"""This tests that cache salts are applied during hashing and the cache
|
||||
is separated cache as expected.
|
||||
"""
|
||||
block_size = 16
|
||||
@@ -2961,8 +2948,7 @@ def test_cache_key_salting():
|
||||
|
||||
|
||||
def test_prefill_not_enough_free_blocks_with_computed_blocks():
|
||||
"""
|
||||
This is a unit test that tests the correctness of the allocate_slots
|
||||
"""This is a unit test that tests the correctness of the allocate_slots
|
||||
when there is not enough free blocks. Specifically, when a request
|
||||
has computed blocks but cannot be allocated due to not enough free blocks,
|
||||
the computed blocks should not be touched.
|
||||
@@ -3344,7 +3330,8 @@ def test_kv_cache_events_with_lora(blocks_to_cache: int):
|
||||
@pytest.mark.parametrize("group_id", [0, 1, 2])
|
||||
def test_block_stored_event_group_idx(group_id: int):
|
||||
"""Test BlockStored events emitted by cache_full_blocks carry the correct
|
||||
group_idx."""
|
||||
group_idx.
|
||||
"""
|
||||
block_size = 4
|
||||
num_tokens = block_size * 2
|
||||
|
||||
@@ -3462,8 +3449,7 @@ def test_session_id_does_not_affect_prefix_cache_identity():
|
||||
|
||||
|
||||
def test_block_stored_event_group_idx_multiple_groups():
|
||||
"""
|
||||
Test BlockStored events for separate HMA groups that each carry the
|
||||
"""Test BlockStored events for separate HMA groups that each carry the
|
||||
correct group_idx.
|
||||
|
||||
Simulates the HMA scenario where full-attention blocks (group 0) and
|
||||
@@ -3585,8 +3571,7 @@ def test_block_stored_event_group_idx_out_of_bounds(monkeypatch):
|
||||
|
||||
@pytest.mark.parametrize("group_id", [0, 1, 2])
|
||||
def test_block_removed_event_group_idx(group_id: int):
|
||||
"""
|
||||
Test BlockRemoved events emitted on eviction carry the group_idx extracted
|
||||
"""Test BlockRemoved events emitted on eviction carry the group_idx extracted
|
||||
from the evicted block's BlockHashWithGroupId via get_group_id().
|
||||
"""
|
||||
block_size = 4
|
||||
@@ -3639,7 +3624,8 @@ def test_block_removed_event_group_idx(group_id: int):
|
||||
def test_emit_cached_block_events():
|
||||
"""emit_cached_block_events emits one BlockStored for already-cached
|
||||
(reused) prefix blocks, carrying the correct group_idx /
|
||||
parent_block_hash / token_ids, and without mutating block state."""
|
||||
parent_block_hash / token_ids, and without mutating block state.
|
||||
"""
|
||||
block_size = 4
|
||||
num_cached_blocks = 3
|
||||
kv_cache_group_id = 1
|
||||
@@ -3752,7 +3738,8 @@ def test_emit_cached_block_events_zero_cached():
|
||||
|
||||
def test_eagle_enabled_removes_last_block():
|
||||
"""Verify Eagle does NOT remove blocks when request
|
||||
length is divisible by block size."""
|
||||
length is divisible by block size.
|
||||
"""
|
||||
block_size = 16
|
||||
manager = make_kv_cache_manager(
|
||||
make_kv_cache_config(block_size, num_blocks=10),
|
||||
@@ -4155,7 +4142,8 @@ def test_hybrid_cache_blocks_swa_tail_window_only():
|
||||
(its right-to-left scan stops once a contiguous match is found). Blocks
|
||||
earlier in the segment can never serve a hit, so
|
||||
``HybridKVCacheCoordinator.cache_blocks`` should skip them rather than
|
||||
polluting the prefix-cache hash map."""
|
||||
polluting the prefix-cache hash map.
|
||||
"""
|
||||
block_size = 8
|
||||
# Full attn block_size=32, SWA block_size=8, sw=8 -> lcm=32.
|
||||
# tail = ceil(7/8) = 1; per_segment = 32/8 = 4.
|
||||
@@ -4226,7 +4214,8 @@ def test_hybrid_cache_blocks_clamped_to_lcm():
|
||||
Chunks past the last lcm-aligned boundary can never participate in a
|
||||
cache hit (find_longest_cache_hit always returns lcm-aligned hits), so
|
||||
caching them only pollutes the prefix-cache hash map and keeps blocks
|
||||
on the LRU list that could otherwise return to the free pool."""
|
||||
on the LRU list that could otherwise return to the free pool.
|
||||
"""
|
||||
block_size = 16
|
||||
# Full attn block_size=32, SWA block_size=16 -> lcm=32.
|
||||
kv_cache_config = KVCacheConfig(
|
||||
@@ -4361,7 +4350,8 @@ def test_hybrid_local_kv_retention_interval_aligns_in_manager():
|
||||
)
|
||||
def test_hybrid_local_kv_retention_interval_rejects_invalid(interval, expected_match):
|
||||
"""A retention interval that is negative or not a multiple of
|
||||
scheduler_block_size errors out at construction time."""
|
||||
scheduler_block_size errors out at construction time.
|
||||
"""
|
||||
block_size = 8
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=100,
|
||||
@@ -4831,7 +4821,8 @@ def test_block_lookup_cache_multi_blocks_per_key():
|
||||
def test_can_fit_full_sequence_swa_cap_admits_long_prompt():
|
||||
"""Hybrid full+SWA model with a pool sized at the startup minimum should
|
||||
admit a prompt longer than the SWA cap, because SlidingWindowManager
|
||||
recycles blocks during chunked prefill (issue #39734)."""
|
||||
recycles blocks during chunked prefill (issue #39734).
|
||||
"""
|
||||
block_size = 16
|
||||
sliding_window = 4 * block_size # 64 tokens
|
||||
max_num_batched_tokens = 8 * block_size # 128 tokens
|
||||
@@ -4926,7 +4917,8 @@ def test_full_sequence_admission_reserves_device_blocks_for_immediate_chunk(
|
||||
|
||||
def test_can_fit_full_sequence_full_attention_still_gates_oversized():
|
||||
"""The cap only loosens the SWA group; a prompt that exceeds the
|
||||
full-attention pool capacity must still be rejected."""
|
||||
full-attention pool capacity must still be rejected.
|
||||
"""
|
||||
block_size = 16
|
||||
sliding_window = 4 * block_size
|
||||
max_num_batched_tokens = 8 * block_size
|
||||
@@ -5045,7 +5037,8 @@ def _take_free_blocks(manager: KVCacheManager, num_blocks: int) -> list[KVCacheB
|
||||
without removing them. These ref_cnt==0 blocks stand in for evictable
|
||||
cache-hit blocks left behind by a previous (e.g. preempted) request, and
|
||||
sitting at the head guarantees a later group's external ``get_new_blocks``
|
||||
would contend for them on unpatched code (issue #33775)."""
|
||||
would contend for them on unpatched code (issue #33775).
|
||||
"""
|
||||
blocks: list[KVCacheBlock] = []
|
||||
head = manager.block_pool.free_block_queue.fake_free_list_head
|
||||
for _ in range(num_blocks):
|
||||
@@ -5056,7 +5049,8 @@ def _take_free_blocks(manager: KVCacheManager, num_blocks: int) -> list[KVCacheB
|
||||
|
||||
def _assert_no_double_allocation(manager: KVCacheManager, req_id: str) -> None:
|
||||
"""No physical block may be handed out twice across groups, and every
|
||||
block referenced by the request must have a live ref_cnt."""
|
||||
block referenced by the request must have a live ref_cnt.
|
||||
"""
|
||||
block_ids = manager.get_blocks(req_id).get_block_ids()
|
||||
flat = [block_id for group in block_ids for block_id in group]
|
||||
assert len(set(flat)) == len(flat), "Block IDs are not unique across groups"
|
||||
@@ -5083,7 +5077,8 @@ def _cross_group_cache_hit(
|
||||
) -> Request:
|
||||
"""Allocate ``req_id`` with a per-group local prefix hit plus external
|
||||
(connector) computed tokens, driving the coordinator's two-phase path.
|
||||
Returns the allocated request so callers can free it (e.g. to preempt)."""
|
||||
Returns the allocated request so callers can free it (e.g. to preempt).
|
||||
"""
|
||||
block_size = _two_phase_block_size(manager)
|
||||
hit_blocks = _take_free_blocks(manager, num_groups * local_blocks_per_group)
|
||||
cache_hit = KVCacheBlocks(
|
||||
@@ -5178,7 +5173,8 @@ def test_swa_free_split_keeps_cached_tail_ahead_of_scratch():
|
||||
"""Dense retention: freeing an SWA request must place its
|
||||
uncached scratch blocks at the front of the free queue (recycled first)
|
||||
and keep its cached checkpoint blocks at the back (retained for prefix
|
||||
hits). This split is always-on, independent of the retention interval."""
|
||||
hits). This split is always-on, independent of the retention interval.
|
||||
"""
|
||||
block_size = 8
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=100,
|
||||
@@ -5289,7 +5285,8 @@ def _make_pure_swa_manager(block_size, sliding_window, num_blocks=100, **kwargs)
|
||||
def test_pure_swa_retention_interval_caches_sparse_tails():
|
||||
"""Sparse retention must work for a pure-SWA single-group model, not just
|
||||
hybrid models: only the per-interval tails plus the latest replay tail are
|
||||
cached, and a replay still hits the latest replayable boundary."""
|
||||
cached, and a replay still hits the latest replayable boundary.
|
||||
"""
|
||||
block_size = 16
|
||||
manager = _make_pure_swa_manager(
|
||||
block_size, sliding_window=block_size, retention_interval=64
|
||||
@@ -5361,7 +5358,8 @@ def test_pure_swa_retention_latest_only():
|
||||
|
||||
def test_pure_swa_dense_retention_caches_all():
|
||||
"""With retention set to ``None``, a pure-SWA model keeps dense behavior:
|
||||
every block boundary is a potential hit, so all blocks are cached."""
|
||||
every block boundary is a potential hit, so all blocks are cached.
|
||||
"""
|
||||
block_size = 16
|
||||
manager = _make_pure_swa_manager(block_size, sliding_window=block_size)
|
||||
|
||||
@@ -5390,7 +5388,8 @@ def test_mamba_reachable_block_mask_sparsifies_retention():
|
||||
"""Mamba state-snapshot retention: with a configured retention interval,
|
||||
the manager keeps one cached state per interval-sized segment (plus the
|
||||
latest replay boundary) instead of a snapshot per block, which is what
|
||||
lets a small attention block_size avoid Mamba dominating the KV pool."""
|
||||
lets a small attention block_size avoid Mamba dominating the KV pool.
|
||||
"""
|
||||
from vllm.v1.core.single_type_kv_cache_manager import MambaManager
|
||||
|
||||
block_size = 16
|
||||
@@ -5428,7 +5427,8 @@ def test_mamba_reachable_block_mask_pins_shared_prefix():
|
||||
"""A Marconi-detected shared prefix (``shared_prefix_boundary``) lands before
|
||||
``num_prompt`` so the replay-boundary rule alone would drop it. The mask must
|
||||
pin the single state block ending on that boundary so sparse retention does
|
||||
not defeat cross-request shared-prefix reuse."""
|
||||
not defeat cross-request shared-prefix reuse.
|
||||
"""
|
||||
from vllm.v1.core.single_type_kv_cache_manager import MambaManager
|
||||
|
||||
block_size = 16
|
||||
@@ -5476,7 +5476,8 @@ def test_mamba_shared_prefix_survives_zero_retention():
|
||||
detection) keeps its Mamba state block cached under
|
||||
``prefix_cache_retention_interval=0``, which otherwise retains only the
|
||||
end-of-prompt replay boundary. Without this, a shared prefix (junction
|
||||
before ``num_prompt``) would be recomputed by every sharing request."""
|
||||
before ``num_prompt``) would be recomputed by every sharing request.
|
||||
"""
|
||||
block_size = 16
|
||||
|
||||
# 16-block (256-token) prompt; replay boundary is block 240 // 16 - 1 = 14.
|
||||
@@ -5518,7 +5519,8 @@ def test_mamba_shared_prefix_reuse_under_zero_retention():
|
||||
detecting request must stay reusable by a later request under
|
||||
``prefix_cache_retention_interval=0``. Without the pin the junction is
|
||||
masked out and the later request misses; with it (and under dense) the reuse
|
||||
is preserved."""
|
||||
is preserved.
|
||||
"""
|
||||
block_size = 16
|
||||
|
||||
def last_req_hit(retention, pin):
|
||||
@@ -5601,7 +5603,8 @@ def test_swa_reachable_block_mask_final_partial_segment():
|
||||
def test_swa_reachable_block_mask_pins_shared_prefix():
|
||||
"""SWA analog of the Mamba pin: the shared-prefix junction must keep the
|
||||
``need``-block sliding-window tail ending on that boundary (not a single
|
||||
block), so a windowed hit can land there under sparse retention."""
|
||||
block), so a windowed hit can land there under sparse retention.
|
||||
"""
|
||||
from vllm.v1.core.single_type_kv_cache_manager import SlidingWindowManager
|
||||
|
||||
block_size = 16
|
||||
@@ -5646,7 +5649,8 @@ def test_swa_reachable_block_mask_pins_shared_prefix():
|
||||
def test_swa_reachable_block_mask_with_dcp_scaling():
|
||||
"""DCP shards each block's KV across ranks, scaling the effective block size.
|
||||
Verify that dcp_world_size > 1 scales block size in reachability calculations,
|
||||
producing different masks than dcp_world_size=1."""
|
||||
producing different masks than dcp_world_size=1.
|
||||
"""
|
||||
from vllm.v1.core.single_type_kv_cache_manager import SlidingWindowManager
|
||||
|
||||
block_size = 16
|
||||
@@ -5705,7 +5709,8 @@ def test_swa_reachable_block_mask_sub_block_alignment_is_dense(
|
||||
multiple of the DCP-scaled block size) cannot be represented exactly. This
|
||||
happens for hybrid offloading (e.g. Gemma), where alignment_tokens is the
|
||||
full-attention chunk size and need not divide the SWA block size. The mask
|
||||
must fall back to dense (None) rather than raise."""
|
||||
must fall back to dense (None) rather than raise.
|
||||
"""
|
||||
from vllm.v1.core.single_type_kv_cache_manager import SlidingWindowManager
|
||||
|
||||
spec = SlidingWindowSpec(
|
||||
@@ -5733,7 +5738,8 @@ def test_mamba_reachable_block_mask_ignores_dcp():
|
||||
"""Mamba uses TP, not DCP: each rank holds the full recurrent state, so a
|
||||
state block spans kv_cache_spec.block_size tokens regardless of DCP. The
|
||||
mask must not scale by dcp_world_size, so retention granularity is
|
||||
identical for any DCP world size."""
|
||||
identical for any DCP world size.
|
||||
"""
|
||||
from vllm.v1.core.single_type_kv_cache_manager import MambaManager
|
||||
|
||||
spec = MambaSpec(
|
||||
@@ -5767,7 +5773,8 @@ def test_mamba_reachable_block_mask_ignores_dcp():
|
||||
|
||||
def test_mamba_reachable_block_mask_large_dcp_stays_sparse():
|
||||
"""A large DCP world size must not scale the Mamba block size, so it can
|
||||
never collapse sparse retention into dense caching."""
|
||||
never collapse sparse retention into dense caching.
|
||||
"""
|
||||
from vllm.v1.core.single_type_kv_cache_manager import MambaManager
|
||||
|
||||
spec = MambaSpec(
|
||||
@@ -5802,7 +5809,8 @@ def test_swa_shared_prefix_reuse_under_zero_retention():
|
||||
"""SWA cross-request analog: a partial shared prefix's sliding-window tail
|
||||
must stay reusable under ``prefix_cache_retention_interval=0``. Without
|
||||
the pin the junction window is masked out and a later request misses; with
|
||||
it (and under dense) reuse is preserved."""
|
||||
it (and under dense) reuse is preserved.
|
||||
"""
|
||||
block_size = 16
|
||||
|
||||
def last_req_hit(retention, pin):
|
||||
|
||||
@@ -14,10 +14,10 @@ pytestmark = pytest.mark.cpu_test
|
||||
|
||||
|
||||
class TestCheckSequenceRepetition:
|
||||
"""Unit tests for the check_sequence_repetition function"""
|
||||
"""Unit tests for the check_sequence_repetition function."""
|
||||
|
||||
def test_simple_repetition_detected(self):
|
||||
"""Test detection of simple repetitive patterns"""
|
||||
"""Test detection of simple repetitive patterns."""
|
||||
token_ids = [1, 2, 3, 1, 2, 3, 1, 2, 3]
|
||||
params = RepetitionDetectionParams(
|
||||
max_pattern_size=3,
|
||||
@@ -27,7 +27,7 @@ class TestCheckSequenceRepetition:
|
||||
assert check_sequence_repetition(token_ids, params)
|
||||
|
||||
def test_repetition_below_min_count(self):
|
||||
"""Test that pattern below min_count is not detected"""
|
||||
"""Test that pattern below min_count is not detected."""
|
||||
token_ids = [1, 2, 3, 1, 2, 3]
|
||||
params = RepetitionDetectionParams(
|
||||
max_pattern_size=3,
|
||||
@@ -37,7 +37,7 @@ class TestCheckSequenceRepetition:
|
||||
assert not check_sequence_repetition(token_ids, params)
|
||||
|
||||
def test_two_token_pattern(self):
|
||||
"""Test detection of 2-token patterns"""
|
||||
"""Test detection of 2-token patterns."""
|
||||
token_ids = [1, 2, 1, 2, 1, 2, 1, 2]
|
||||
params = RepetitionDetectionParams(
|
||||
max_pattern_size=5,
|
||||
@@ -47,7 +47,7 @@ class TestCheckSequenceRepetition:
|
||||
assert check_sequence_repetition(token_ids, params)
|
||||
|
||||
def test_no_repetition_varied_sequence(self):
|
||||
"""Test that non-repetitive sequences are not flagged"""
|
||||
"""Test that non-repetitive sequences are not flagged."""
|
||||
token_ids = [1, 2, 3, 4, 5, 6, 7, 8, 9]
|
||||
params = RepetitionDetectionParams(
|
||||
max_pattern_size=5,
|
||||
@@ -57,7 +57,7 @@ class TestCheckSequenceRepetition:
|
||||
assert not check_sequence_repetition(token_ids, params)
|
||||
|
||||
def test_partial_repetition_not_detected(self):
|
||||
"""Test that incomplete repetitions are not detected"""
|
||||
"""Test that incomplete repetitions are not detected."""
|
||||
token_ids = [1, 2, 3, 1, 2, 3, 1, 2, 4]
|
||||
params = RepetitionDetectionParams(
|
||||
max_pattern_size=3,
|
||||
@@ -67,7 +67,7 @@ class TestCheckSequenceRepetition:
|
||||
assert not check_sequence_repetition(token_ids, params)
|
||||
|
||||
def test_empty_token_list(self):
|
||||
"""Test with empty token list"""
|
||||
"""Test with empty token list."""
|
||||
params = RepetitionDetectionParams(
|
||||
max_pattern_size=3,
|
||||
min_pattern_size=2,
|
||||
@@ -76,19 +76,19 @@ class TestCheckSequenceRepetition:
|
||||
assert not check_sequence_repetition([], params)
|
||||
|
||||
def test_detection_disabled_max_size_zero(self):
|
||||
"""Test that zero max_pattern_size disables detection"""
|
||||
"""Test that zero max_pattern_size disables detection."""
|
||||
token_ids = [1, 2, 1, 2, 1, 2]
|
||||
params = RepetitionDetectionParams()
|
||||
assert not check_sequence_repetition(token_ids, params)
|
||||
|
||||
def test_invalid_min_count(self):
|
||||
"""Test that min_count < 2 returns False"""
|
||||
"""Test that min_count < 2 returns False."""
|
||||
token_ids = [1, 2, 1, 2]
|
||||
params = RepetitionDetectionParams()
|
||||
assert not check_sequence_repetition(token_ids, params)
|
||||
|
||||
def test_repetition_at_end_of_sequence(self):
|
||||
"""Test detection when repetition occurs at the end"""
|
||||
"""Test detection when repetition occurs at the end."""
|
||||
token_ids = [1, 2, 3, 4, 5, 6, 5, 6, 5, 6]
|
||||
params = RepetitionDetectionParams(
|
||||
max_pattern_size=3,
|
||||
@@ -98,7 +98,7 @@ class TestCheckSequenceRepetition:
|
||||
assert check_sequence_repetition(token_ids, params)
|
||||
|
||||
def test_large_pattern_many_repetitions(self):
|
||||
"""Test large pattern repeated many times"""
|
||||
"""Test large pattern repeated many times."""
|
||||
token_ids = [1, 2, 3, 4, 5, 6, 7, 8] * 5
|
||||
params = RepetitionDetectionParams(
|
||||
max_pattern_size=10,
|
||||
@@ -114,10 +114,10 @@ class TestCheckSequenceRepetition:
|
||||
|
||||
|
||||
class TestRepetitionDetectionIntegration:
|
||||
"""Integration tests for repetition detection in check_stop"""
|
||||
"""Integration tests for repetition detection in check_stop."""
|
||||
|
||||
def test_basic_repetition_stops_generation(self):
|
||||
"""Test that repetition is detected and stops generation"""
|
||||
"""Test that repetition is detected and stops generation."""
|
||||
params = SamplingParams(
|
||||
max_tokens=100,
|
||||
repetition_detection=RepetitionDetectionParams(
|
||||
@@ -138,7 +138,7 @@ class TestRepetitionDetectionIntegration:
|
||||
assert request.stop_reason == "repetition_detected"
|
||||
|
||||
def test_detection_disabled_no_stop(self):
|
||||
"""Test that disabled detection doesn't stop generation"""
|
||||
"""Test that disabled detection doesn't stop generation."""
|
||||
params = SamplingParams(
|
||||
max_tokens=100,
|
||||
)
|
||||
@@ -152,7 +152,7 @@ class TestRepetitionDetectionIntegration:
|
||||
assert not check_stop(request, max_model_len=1024)
|
||||
|
||||
def test_repetition_respects_min_tokens(self):
|
||||
"""Test that repetition detection respects min_tokens"""
|
||||
"""Test that repetition detection respects min_tokens."""
|
||||
params = SamplingParams(
|
||||
min_tokens=10,
|
||||
max_tokens=100,
|
||||
@@ -172,7 +172,7 @@ class TestRepetitionDetectionIntegration:
|
||||
assert not check_stop(request, max_model_len=1024)
|
||||
|
||||
def test_no_repetition_continues_generation(self):
|
||||
"""Test that non-repetitive tokens don't stop generation"""
|
||||
"""Test that non-repetitive tokens don't stop generation."""
|
||||
params = SamplingParams(
|
||||
max_tokens=100,
|
||||
repetition_detection=RepetitionDetectionParams(
|
||||
@@ -191,7 +191,7 @@ class TestRepetitionDetectionIntegration:
|
||||
assert not check_stop(request, max_model_len=1024)
|
||||
|
||||
def test_pattern_at_size_boundary(self):
|
||||
"""Test detection at exact pattern size boundary"""
|
||||
"""Test detection at exact pattern size boundary."""
|
||||
params = SamplingParams(
|
||||
max_tokens=100,
|
||||
repetition_detection=RepetitionDetectionParams(
|
||||
@@ -211,7 +211,7 @@ class TestRepetitionDetectionIntegration:
|
||||
assert request.status == RequestStatus.FINISHED_REPETITION
|
||||
|
||||
def test_multiple_pattern_sizes_checked(self):
|
||||
"""Test that function checks pattern sizes in range"""
|
||||
"""Test that function checks pattern sizes in range."""
|
||||
params = SamplingParams(
|
||||
max_tokens=100,
|
||||
repetition_detection=RepetitionDetectionParams(
|
||||
@@ -231,7 +231,7 @@ class TestRepetitionDetectionIntegration:
|
||||
assert request.status == RequestStatus.FINISHED_REPETITION
|
||||
|
||||
def test_eos_takes_precedence_over_repetition(self):
|
||||
"""Test that EOS token stops before repetition check"""
|
||||
"""Test that EOS token stops before repetition check."""
|
||||
params = SamplingParams(
|
||||
max_tokens=100,
|
||||
stop_token_ids=[999],
|
||||
@@ -252,7 +252,7 @@ class TestRepetitionDetectionIntegration:
|
||||
assert request.status == RequestStatus.FINISHED_STOPPED
|
||||
|
||||
def test_min_pattern_size_filters_small_patterns(self):
|
||||
"""Test that min_pattern_size filters out smaller patterns"""
|
||||
"""Test that min_pattern_size filters out smaller patterns."""
|
||||
params = SamplingParams(
|
||||
max_tokens=100,
|
||||
repetition_detection=RepetitionDetectionParams(
|
||||
@@ -271,7 +271,7 @@ class TestRepetitionDetectionIntegration:
|
||||
assert not check_stop(request, max_model_len=1024)
|
||||
|
||||
def test_high_repetition_threshold(self):
|
||||
"""Test that high min_count requires many repetitions"""
|
||||
"""Test that high min_count requires many repetitions."""
|
||||
params = SamplingParams(
|
||||
max_tokens=100,
|
||||
repetition_detection=RepetitionDetectionParams(
|
||||
|
||||
@@ -295,7 +295,7 @@ def test_schedule_multimodal_requests():
|
||||
|
||||
|
||||
def test_async_scheduling_pp_allows_rescheduling_with_output_placeholders():
|
||||
"""Async scheduling + PP: allow multi-step in-flight scheduling per request"""
|
||||
"""Async scheduling + PP: allow multi-step in-flight scheduling per request."""
|
||||
scheduler = create_scheduler(async_scheduling=True, pipeline_parallel_size=2)
|
||||
(req,) = create_requests(num_requests=1, num_tokens=8)
|
||||
scheduler.add_request(req)
|
||||
@@ -538,7 +538,8 @@ def test_throttle_defers_inflight_prefill_chunk():
|
||||
"""DP prefill balancing throttles ALL prefill compute on a throttled step,
|
||||
not just new admissions: an in-progress (chunked) prefill already in the
|
||||
running queue is also deferred, so the step runs decode-only, while a
|
||||
separate decode keeps being scheduled."""
|
||||
separate decode keeps being scheduled.
|
||||
"""
|
||||
scheduler = create_scheduler(
|
||||
max_num_seqs=16, max_num_batched_tokens=50, enable_chunked_prefill=True
|
||||
)
|
||||
@@ -592,7 +593,8 @@ def test_throttle_capacity_bound_guard_admits():
|
||||
"""Saturation guard: if a cadence-aligned release step cannot drain the
|
||||
waiting prefill queue (it ran out of token budget), the throttle backs off on
|
||||
the next step so the backlog cannot grow into a TTFT avalanche -- prefills are
|
||||
admitted even though throttle_prefills is set."""
|
||||
admitted even though throttle_prefills is set.
|
||||
"""
|
||||
scheduler = create_scheduler(
|
||||
max_num_seqs=16, max_num_batched_tokens=200, enable_chunked_prefill=True
|
||||
)
|
||||
@@ -785,7 +787,7 @@ def test_update_from_output_routes_sampling_masks_by_request():
|
||||
|
||||
|
||||
def test_stop_via_update_from_output():
|
||||
"""Test stopping behavior through update_from_output"""
|
||||
"""Test stopping behavior through update_from_output."""
|
||||
scheduler = create_scheduler(num_speculative_tokens=1)
|
||||
|
||||
# Test case 1: Stop on EOS token
|
||||
@@ -1291,7 +1293,8 @@ def test_pending_hisparse_reclamation_defers_preemption(monkeypatch):
|
||||
|
||||
def test_prefix_cache_query_not_inflated_by_connector_defer():
|
||||
"""The GPU prefix-cache query is recorded at admission, so a request the
|
||||
connector defers several times is counted once, not once per retry."""
|
||||
connector defers several times is counted once, not once per retry.
|
||||
"""
|
||||
num_defers_before_matching = 3
|
||||
scheduler = create_scheduler(
|
||||
enable_prefix_caching=True,
|
||||
@@ -1319,7 +1322,8 @@ def test_prefix_cache_query_not_inflated_by_connector_defer():
|
||||
|
||||
def test_preemption_re_records_prefix_cache_query():
|
||||
"""A preempted request re-enters the lookup on resume, so its recomputation
|
||||
is counted again into the preempted stats."""
|
||||
is counted again into the preempted stats.
|
||||
"""
|
||||
scheduler = create_scheduler(enable_prefix_caching=True)
|
||||
request = create_requests(num_requests=1)[0]
|
||||
scheduler.add_request(request)
|
||||
@@ -1355,7 +1359,8 @@ def test_preemption_re_records_prefix_cache_query():
|
||||
|
||||
def test_prefix_cache_stats_not_recorded_when_caching_disabled():
|
||||
"""With prefix caching off there is no local lookup, so admitting a request
|
||||
records no phantom miss."""
|
||||
records no phantom miss.
|
||||
"""
|
||||
scheduler = create_scheduler(enable_prefix_caching=False)
|
||||
for request in create_requests(num_requests=2):
|
||||
scheduler.add_request(request)
|
||||
@@ -1369,7 +1374,8 @@ def test_prefix_cache_stats_not_recorded_when_caching_disabled():
|
||||
|
||||
def test_prefix_cache_stats_counted_once_for_retried_then_scheduled_request():
|
||||
"""A real cache hit rejected once by allocate_slots and admitted on the next
|
||||
step is counted exactly once, hits included."""
|
||||
step is counted exactly once, hits included.
|
||||
"""
|
||||
block_size = 16
|
||||
scheduler = create_scheduler(
|
||||
enable_prefix_caching=True,
|
||||
@@ -2130,7 +2136,6 @@ def _assert_right_scheduler_output(
|
||||
expected_num_scheduled_tokens: int,
|
||||
):
|
||||
"""Check if SchedulerOutput is correct after remote KV cache hit."""
|
||||
|
||||
# We should inject the kv_connector_metadata.
|
||||
assert len(output.kv_connector_metadata.requests) == num_requests
|
||||
|
||||
@@ -2148,7 +2153,6 @@ def _assert_right_kv_cache_manager(
|
||||
num_total_blocks: int,
|
||||
):
|
||||
"""Check whether KVCacheManager is correct after allocate."""
|
||||
|
||||
# Make sure the request stats are right.
|
||||
EXPECTED_TOTAL_BLOCKS = num_tokens // block_size
|
||||
for req in requests:
|
||||
@@ -2179,7 +2183,6 @@ def _step_until_done(
|
||||
model_runner_output: ModelRunnerOutput,
|
||||
):
|
||||
"""Loop over schedule(), update_from_output() until finished."""
|
||||
|
||||
all_finished = False
|
||||
_ = scheduler.update_from_output(output, model_runner_output)
|
||||
while not all_finished:
|
||||
@@ -2207,7 +2210,6 @@ def _num_waiting_requests(scheduler: Scheduler) -> int:
|
||||
|
||||
def _step_until_kv_transfer_finished(scheduler: Scheduler, req_ids: list[str]):
|
||||
"""Cycle requests through a KV transfer cycle."""
|
||||
|
||||
# Requests should first transition to WAITING_FOR_REMOTE_KVS
|
||||
output = scheduler.schedule()
|
||||
assert _num_waiting_requests(scheduler) == len(req_ids)
|
||||
@@ -2291,11 +2293,9 @@ def test_has_sync_kv_loads(
|
||||
|
||||
@pytest.mark.parametrize("is_async", [False, True])
|
||||
def test_kv_connector_basic(is_async: bool):
|
||||
"""
|
||||
Test whether Scheduler with KVConnector schedules tokens, allocates
|
||||
"""Test whether Scheduler with KVConnector schedules tokens, allocates
|
||||
memory, and cleans up requests as expected under normal operation.
|
||||
"""
|
||||
|
||||
# Setup Scheduler.
|
||||
BLOCK_SIZE = 16
|
||||
NUM_MATCHED_NEW_TOKENS = BLOCK_SIZE * 2
|
||||
@@ -2420,11 +2420,9 @@ def test_kv_connector_basic(is_async: bool):
|
||||
@pytest.mark.parametrize("is_async", [False, True])
|
||||
@pytest.mark.parametrize("local_cache_hits", [False, True])
|
||||
def test_external_prefix_cache_metrics(is_async: bool, local_cache_hits: bool):
|
||||
"""
|
||||
Verify connector prefix cache metrics are updated
|
||||
"""Verify connector prefix cache metrics are updated
|
||||
correctly when the scheduler processes requests with KV connector hits.
|
||||
"""
|
||||
|
||||
BLOCK_SIZE = 16
|
||||
if local_cache_hits:
|
||||
NUM_MATCHED_NEW_TOKENS = BLOCK_SIZE * 2 # 32 tokens
|
||||
@@ -2540,11 +2538,9 @@ def test_external_prefix_cache_metrics(is_async: bool, local_cache_hits: bool):
|
||||
"use_ec_connector, ec_role", [(False, None), (True, "ec_consumer")]
|
||||
)
|
||||
def test_kv_connector_unable_to_allocate(use_ec_connector, ec_role):
|
||||
"""
|
||||
Test whether scheduler with KVConnector is able to handle
|
||||
"""Test whether scheduler with KVConnector is able to handle
|
||||
unable to allocate (run out of blocks in allocate_slots().
|
||||
"""
|
||||
|
||||
# Setup Scheduler With Mock External Cache Hit.
|
||||
BLOCK_SIZE = 4
|
||||
NUM_BLOCKS = 10
|
||||
@@ -2627,11 +2623,9 @@ def test_kv_connector_unable_to_allocate(use_ec_connector, ec_role):
|
||||
def test_kv_connector_handles_preemption(
|
||||
is_async, use_ec_connector, ec_role, use_v2_model_runner
|
||||
):
|
||||
"""
|
||||
Test whether scheduler with KVConnector is able to handle
|
||||
"""Test whether scheduler with KVConnector is able to handle
|
||||
unable to allocate (run out of blocks in allocate_slots().
|
||||
"""
|
||||
|
||||
# Setup Scheduler With Mock External Cache Hit.
|
||||
BLOCK_SIZE = 2
|
||||
# NOTE: there is 1 null block, so this is 6 blocks.
|
||||
@@ -2842,7 +2836,6 @@ def assert_scheduler_empty(scheduler: Scheduler):
|
||||
|
||||
def test_memory_leak():
|
||||
"""Test that we do not have a memory leak."""
|
||||
|
||||
scheduler = create_scheduler(enable_prefix_caching=True)
|
||||
|
||||
NUM_REQUESTS = 5
|
||||
@@ -2899,6 +2892,7 @@ def create_scheduler_with_priority(
|
||||
|
||||
Returns:
|
||||
{class}`Scheduler` instance with priority scheduling
|
||||
|
||||
"""
|
||||
model_config = ModelConfig(
|
||||
model=model,
|
||||
@@ -3100,7 +3094,8 @@ def create_requests_with_priority(
|
||||
|
||||
def test_priority_scheduling_basic_ordering():
|
||||
"""Test that requests are scheduled in priority order
|
||||
(lower value = higher priority)."""
|
||||
(lower value = higher priority).
|
||||
"""
|
||||
scheduler = create_scheduler_with_priority()
|
||||
|
||||
# Create requests with different priorities
|
||||
@@ -3129,7 +3124,8 @@ def test_priority_scheduling_basic_ordering():
|
||||
|
||||
def test_priority_scheduling_arrival_time_tiebreaker():
|
||||
"""Test that arrival time is used
|
||||
as tiebreaker when priorities are equal."""
|
||||
as tiebreaker when priorities are equal.
|
||||
"""
|
||||
scheduler = create_scheduler_with_priority()
|
||||
|
||||
# Create requests with same priority but different arrival times
|
||||
@@ -3286,7 +3282,8 @@ def test_priority_scheduling_preemption():
|
||||
|
||||
def test_priority_scheduling_no_preemption_when_space_available():
|
||||
"""Test that preemption doesn't happen
|
||||
when there's space for new requests."""
|
||||
when there's space for new requests.
|
||||
"""
|
||||
scheduler = create_scheduler_with_priority(
|
||||
max_num_seqs=3, # Allow 3 concurrent requests
|
||||
max_num_batched_tokens=200, # Sufficient token budget
|
||||
@@ -3339,7 +3336,8 @@ def test_priority_scheduling_no_preemption_when_space_available():
|
||||
|
||||
def test_priority_scheduling_preemption_victim_selection():
|
||||
"""Test that the correct victim is selected for
|
||||
preemption based on priority and arrival time."""
|
||||
preemption based on priority and arrival time.
|
||||
"""
|
||||
# This test verifies the priority-based victim selection logic
|
||||
# by checking the waiting queue order after adding requests with different
|
||||
# priorities
|
||||
@@ -3459,7 +3457,8 @@ def test_priority_scheduling_waiting_queue_order():
|
||||
|
||||
def test_priority_scheduling_fcfs_fallback():
|
||||
"""Test that FCFS behavior is maintained when all
|
||||
requests have same priority."""
|
||||
requests have same priority.
|
||||
"""
|
||||
scheduler = create_scheduler_with_priority()
|
||||
|
||||
# Create requests with same priority but different arrival times
|
||||
@@ -3528,7 +3527,8 @@ def test_priority_scheduling_with_limited_slots():
|
||||
|
||||
def test_priority_scheduling_heap_property():
|
||||
"""Test that the waiting queue maintains heap
|
||||
property for priority scheduling."""
|
||||
property for priority scheduling.
|
||||
"""
|
||||
scheduler = create_scheduler_with_priority(
|
||||
max_num_seqs=1, # Only one request can run at a time
|
||||
)
|
||||
@@ -3753,7 +3753,8 @@ def test_priority_scheduling_preemption_and_resumption_when_out_of_kv(
|
||||
use_ec_connector, ec_role, use_v2_model_runner
|
||||
):
|
||||
"""Test that priority scheduling preempts lower priority requests
|
||||
when out of KV cache space."""
|
||||
when out of KV cache space.
|
||||
"""
|
||||
# Create scheduler with very limited memory to force preemption
|
||||
scheduler = create_scheduler_with_priority(
|
||||
max_num_seqs=2, # Allow multiple requests
|
||||
@@ -3898,7 +3899,8 @@ def test_chunked_prefill_disabled_for_encoder_decoder(
|
||||
enable_chunked_prefill: bool, is_encoder_decoder: bool, expect_enabled: bool
|
||||
) -> None:
|
||||
"""Validate that chunked prefill is appropriately disabled for
|
||||
encoder-decoder models."""
|
||||
encoder-decoder models.
|
||||
"""
|
||||
scheduler_config = SchedulerConfig(
|
||||
enable_chunked_prefill=enable_chunked_prefill,
|
||||
is_encoder_decoder=is_encoder_decoder,
|
||||
@@ -3927,7 +3929,8 @@ def _validate_chunked_prefill_settings_for_encoder_decoder(
|
||||
scheduler_config: SchedulerConfig, is_encoder_decoder: bool, expect_enabled: bool
|
||||
) -> None:
|
||||
"""Validate chunked prefill settings in the scheduler config for
|
||||
encoder-decoder models."""
|
||||
encoder-decoder models.
|
||||
"""
|
||||
assert scheduler_config.enable_chunked_prefill is expect_enabled
|
||||
if is_encoder_decoder:
|
||||
# Encoder-decoder models should automatically disable chunked multimodal
|
||||
@@ -3987,7 +3990,7 @@ def _assert_right_ec_connector_metadata(
|
||||
output: SchedulerOutput,
|
||||
mm_features_list: list[MultiModalFeatureSpec],
|
||||
):
|
||||
"""Verify that ECConnector metadata EXACTLY matches the input MM data"""
|
||||
"""Verify that ECConnector metadata EXACTLY matches the input MM data."""
|
||||
# Get the connector metadata
|
||||
metadata = output.ec_connector_metadata
|
||||
|
||||
@@ -4017,8 +4020,8 @@ def _assert_right_encoder_inputs(
|
||||
):
|
||||
"""Verify that requests/mm_hashes should (not) in scheduled encoder input
|
||||
If check_exist is False, this function returns True
|
||||
if requests are NOT in encoder inputs"""
|
||||
|
||||
if requests are NOT in encoder inputs
|
||||
"""
|
||||
# Get the scheduled encoder inputs
|
||||
# NOTE: scheduled_encoder_inputs is a dictionary with request id as key
|
||||
scheduled_encoder_inputs = output.scheduled_encoder_inputs
|
||||
@@ -4179,7 +4182,8 @@ def test_ec_connector_text_only_request(use_kv_connector):
|
||||
@pytest.mark.parametrize("use_kv_connector", [False, True])
|
||||
def test_ec_connector_cache_hit_external_load(use_kv_connector):
|
||||
"""Test ec_consumer loads from external cache when hit.
|
||||
A normal basic operation for EPD disaggrgation"""
|
||||
A normal basic operation for EPD disaggrgation
|
||||
"""
|
||||
scheduler = create_scheduler(
|
||||
model="llava-hf/llava-1.5-7b-hf",
|
||||
enable_prefix_caching=True,
|
||||
@@ -4527,11 +4531,9 @@ def test_ec_connector_schedule_multiple_requests(cache_exist, use_kv_connector):
|
||||
|
||||
@pytest.mark.parametrize("use_kv_connector", [False, True])
|
||||
def test_ec_connector_unable_to_allocate(use_kv_connector):
|
||||
"""
|
||||
Test whether scheduler with ECConnector is able to handle
|
||||
"""Test whether scheduler with ECConnector is able to handle
|
||||
unable to allocate (run out of blocks).
|
||||
"""
|
||||
|
||||
# Setup Scheduler With Mock External Cache Hit.
|
||||
BLOCK_SIZE = 4
|
||||
NUM_BLOCKS = 10
|
||||
@@ -4628,7 +4630,8 @@ def test_priority_scheduling_ec_connector_preemption_and_resumption(
|
||||
cache_exist, use_kv_connector
|
||||
):
|
||||
"""Test that priority scheduling preempts lower priority requests
|
||||
when out of KV cache space."""
|
||||
when out of KV cache space.
|
||||
"""
|
||||
# Create scheduler with very limited memory to force preemption
|
||||
scheduler = create_scheduler_with_priority(
|
||||
model="llava-hf/llava-1.5-7b-hf",
|
||||
@@ -4860,8 +4863,7 @@ def test_priority_scheduling_ec_connector_preemption_and_resumption(
|
||||
|
||||
@pytest.mark.parametrize("use_kv_connector", [False, True])
|
||||
def test_ec_connector_allocate_encoder_tokens_with_external_load(use_kv_connector):
|
||||
"""
|
||||
Scenario:
|
||||
"""Scenario:
|
||||
- Encoder cache size: 32
|
||||
- Request A: 1 feature (12 tokens) → NOT cached remotely.
|
||||
- Request B: 3 features (3 x 10 tokens) → ALL cached remotely.
|
||||
@@ -5298,7 +5300,8 @@ def test_scheduler_kv_connector_stats():
|
||||
|
||||
def test_ec_connector_update_connector_output_called():
|
||||
"""Test that worker-side EC connector output is forwarded to the
|
||||
EC connector's update_connector_output hook."""
|
||||
EC connector's update_connector_output hook.
|
||||
"""
|
||||
scheduler = create_scheduler(
|
||||
model="llava-hf/llava-1.5-7b-hf",
|
||||
use_ec_connector=True,
|
||||
@@ -5503,7 +5506,8 @@ def test_variable_length_cross_attn_block_allocation():
|
||||
|
||||
def test_cross_attn_blocks_not_over_allocated():
|
||||
"""Test that cross-attention blocks are not over-allocated compared to
|
||||
what each request actually needs."""
|
||||
what each request actually needs.
|
||||
"""
|
||||
from math import ceil
|
||||
|
||||
block_size = 16
|
||||
@@ -5599,7 +5603,8 @@ def test_cross_attn_blocks_not_under_allocated():
|
||||
|
||||
def test_cross_attn_zero_blocks_without_encoder_inputs():
|
||||
"""Test that requests without encoder inputs get zero cross-attention
|
||||
blocks, even when the scheduler is configured for encoder-decoder."""
|
||||
blocks, even when the scheduler is configured for encoder-decoder.
|
||||
"""
|
||||
block_size = 16
|
||||
scheduler = _create_encoder_decoder_scheduler(block_size=block_size)
|
||||
|
||||
@@ -5704,7 +5709,8 @@ def test_free_encoder_inputs_respects_unconfirmed_placeholders():
|
||||
evicted entry and crashes the engine with "Encoder cache miss". The
|
||||
scheduler must retain the input until the *confirmed* progress
|
||||
(num_computed_tokens - num_output_placeholders) passes the range end, so
|
||||
that no pending rejection can rewind into the range."""
|
||||
that no pending rejection can rewind into the range.
|
||||
"""
|
||||
scheduler = create_scheduler(
|
||||
model="llava-hf/llava-1.5-7b-hf",
|
||||
num_speculative_tokens=3,
|
||||
@@ -5815,7 +5821,8 @@ def test_free_encoder_inputs_defers_for_eagle_lookahead():
|
||||
position so the drafter's +1 look-ahead mm-embedding gather (which reads one
|
||||
position past the target's computed range) still finds it cached. This is
|
||||
the primary mechanism that prevents the drafter "Encoder cache miss"; the
|
||||
worker-side token-embedding fallback is only a backstop."""
|
||||
worker-side token-embedding fallback is only a backstop.
|
||||
"""
|
||||
scheduler = create_scheduler(model="llava-hf/llava-1.5-7b-hf")
|
||||
# create_scheduler only builds ngram spec configs; force the eagle path that
|
||||
# _free_encoder_inputs keys off (its read-ahead deferral).
|
||||
@@ -5845,7 +5852,8 @@ def test_free_encoder_inputs_defers_for_eagle_lookahead():
|
||||
|
||||
def test_free_encoder_inputs_unchanged_without_spec_decode():
|
||||
"""Without speculative decoding, encoder inputs are freed as soon as
|
||||
num_computed_tokens passes the placeholder range, as before."""
|
||||
num_computed_tokens passes the placeholder range, as before.
|
||||
"""
|
||||
scheduler = create_scheduler(model="llava-hf/llava-1.5-7b-hf")
|
||||
mm_positions = [[PlaceholderRange(offset=50, length=100)]]
|
||||
request = create_requests(
|
||||
@@ -5876,7 +5884,8 @@ def test_encoder_cache_retained_across_preemption_and_resume():
|
||||
encoder input must pull the still-cached entry back out of `freeable`
|
||||
without scheduling a recompute, keeping the scheduler and worker
|
||||
consistent. The spec-rollback retention margin does not gate this path,
|
||||
so it is covered separately here."""
|
||||
so it is covered separately here.
|
||||
"""
|
||||
scheduler = create_scheduler(
|
||||
model="llava-hf/llava-1.5-7b-hf",
|
||||
num_speculative_tokens=3,
|
||||
@@ -5920,7 +5929,8 @@ def test_encoder_cache_recomputed_when_evicted_during_preemption():
|
||||
pressure before it resumes, the scheduler reports the mm_hash as freed
|
||||
(so the worker drops it) and a resume must schedule a recompute rather
|
||||
than assume the worker still holds it. check_and_update_cache must
|
||||
return False so the encoder input is re-scheduled."""
|
||||
return False so the encoder input is re-scheduled.
|
||||
"""
|
||||
scheduler = create_scheduler(
|
||||
model="llava-hf/llava-1.5-7b-hf",
|
||||
num_speculative_tokens=3,
|
||||
@@ -6571,7 +6581,8 @@ def _decode_ready_request(scheduler):
|
||||
|
||||
def test_update_draft_token_ids_strips_ngram_padding():
|
||||
"""ngram_gpu pads unfilled draft slots with -1; manager validation must
|
||||
strip them before grammar.validate_tokens (which otherwise raises)."""
|
||||
strip them before grammar.validate_tokens (which otherwise raises).
|
||||
"""
|
||||
scheduler = create_scheduler(num_speculative_tokens=4)
|
||||
request = _decode_ready_request(scheduler)
|
||||
|
||||
@@ -6590,7 +6601,8 @@ def test_update_draft_token_ids_strips_ngram_padding():
|
||||
|
||||
def test_update_draft_token_ids_in_output_strips_padding():
|
||||
"""Same guard on the output path; the -1 pad-back for the rejected count
|
||||
is preserved (only the input to manager validation is stripped)."""
|
||||
is preserved (only the input to manager validation is stripped).
|
||||
"""
|
||||
scheduler = create_scheduler(num_speculative_tokens=4)
|
||||
request = _decode_ready_request(scheduler)
|
||||
|
||||
|
||||
@@ -97,7 +97,8 @@ def test_num_in_flight_tokens_accounting():
|
||||
|
||||
def test_swa_free_waits_for_in_flight_step():
|
||||
"""Async: out-of-window blocks stay allocated until the step that still
|
||||
reads them has been processed."""
|
||||
reads them has been processed.
|
||||
"""
|
||||
scheduler = _create_swa_scheduler(async_scheduling=True)
|
||||
request = create_requests(
|
||||
num_requests=1, num_tokens=NUM_PROMPT_TOKENS, block_size=BLOCK_SIZE
|
||||
@@ -132,7 +133,8 @@ def test_swa_free_waits_for_in_flight_step():
|
||||
|
||||
def test_swa_free_immediate_when_sync():
|
||||
"""Sync: no in-flight step at schedule time, frees happen at the first
|
||||
decode allocation as before."""
|
||||
decode allocation as before.
|
||||
"""
|
||||
scheduler = _create_swa_scheduler(async_scheduling=False)
|
||||
request = create_requests(
|
||||
num_requests=1, num_tokens=NUM_PROMPT_TOKENS, block_size=BLOCK_SIZE
|
||||
@@ -171,7 +173,8 @@ def test_swa_admission_cap_accounts_for_overlapping_batches():
|
||||
def test_chunked_local_free_waits_for_in_flight_step():
|
||||
"""Chunked-local attention frees whole chunks left of the current one, and
|
||||
is exposed to the same load-WAR: with async scheduling those chunks must
|
||||
stay allocated until the in-flight step that still reads them settles."""
|
||||
stay allocated until the in-flight step that still reads them settles.
|
||||
"""
|
||||
scheduler = _create_chunked_scheduler(async_scheduling=True)
|
||||
request = create_requests(
|
||||
num_requests=1, num_tokens=NUM_PROMPT_TOKENS, block_size=BLOCK_SIZE
|
||||
@@ -226,7 +229,8 @@ def test_connector_finish_frees_on_settled_basis():
|
||||
is handed to a KV connector (simple CPU offload / NIXL store), must use the
|
||||
same processed-token basis. Otherwise the connector reads/hands off a block
|
||||
the still-in-flight step is optimistically counted as done with, which is
|
||||
the load-path WAR this fix closes."""
|
||||
the load-path WAR this fix closes.
|
||||
"""
|
||||
scheduler = create_scheduler(
|
||||
block_size=BLOCK_SIZE,
|
||||
async_scheduling=True,
|
||||
|
||||
@@ -89,6 +89,7 @@ def create_scheduler(
|
||||
|
||||
Returns:
|
||||
{class}`Scheduler` instance
|
||||
|
||||
"""
|
||||
model_config = ModelConfig(
|
||||
model=model,
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Unit tests for the breakable cudagraph primitives.
|
||||
"""
|
||||
"""Unit tests for the breakable cudagraph primitives."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -82,7 +80,8 @@ def test_piecewise_capture_builds_fresh_metadata_for_both_passes():
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_breakable_tls():
|
||||
"""Defensively clear thread-local capture state between tests so a
|
||||
failure in one test can't leak "nested capture" errors into the next."""
|
||||
failure in one test can't leak "nested capture" errors into the next.
|
||||
"""
|
||||
from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphCapture
|
||||
|
||||
BreakableCUDAGraphCapture._tls.active = None
|
||||
@@ -292,7 +291,8 @@ def test_capture_replay_matches_eager_simple(cuda_capture_stream):
|
||||
def test_decorator_breaks_when_invoked_inside_capture(cuda_capture_stream):
|
||||
"""Verify @eager_break_during_capture correctly routes through
|
||||
add_eager when inside a capture context, and runs straight through
|
||||
when there's no active capture."""
|
||||
when there's no active capture.
|
||||
"""
|
||||
from vllm.compilation.breakable_cudagraph import (
|
||||
BreakableCUDAGraphCapture,
|
||||
eager_break_during_capture,
|
||||
@@ -554,7 +554,8 @@ def _mock_vllm_config():
|
||||
|
||||
def test_wrapper_falls_through_on_runtime_mode_mismatch():
|
||||
"""A PIECEWISE-scoped wrapper runs other modes eagerly, capturing
|
||||
nothing, so an outer CUDAGraphWrapper can own them."""
|
||||
nothing, so an outer CUDAGraphWrapper can own them.
|
||||
"""
|
||||
from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphWrapper
|
||||
from vllm.config import CUDAGraphMode
|
||||
from vllm.forward_context import BatchDescriptor, set_forward_context
|
||||
|
||||
@@ -376,7 +376,6 @@ def _run_and_monitor_call(
|
||||
wrapper, input_tensor, runtime_mode, batch_descriptor, vllm_config
|
||||
):
|
||||
"""Helper to run a single call and monitor the action."""
|
||||
|
||||
with (
|
||||
patch("torch.cuda.graph", wraps=torch.cuda.graph) as mock_graph_context,
|
||||
patch.object(wrapper, "runnable", wraps=wrapper.runnable) as mock_runnable,
|
||||
|
||||
@@ -31,8 +31,7 @@ def test_v1_generation_is_deterministic_across_batch_sizes_with_needle(
|
||||
backend,
|
||||
rms_norm_impl,
|
||||
):
|
||||
"""
|
||||
Ensures that the same request (the 'needle' prompt) yields identical output
|
||||
"""Ensures that the same request (the 'needle' prompt) yields identical output
|
||||
whether run alone (bs=1) or mixed into a larger batch (e.g., bs=64),
|
||||
using the high-level v1 LLM() API only (no manual batching).
|
||||
|
||||
@@ -51,6 +50,7 @@ def test_v1_generation_is_deterministic_across_batch_sizes_with_needle(
|
||||
to produce a more random-sounding phrase, yet remain deterministic by
|
||||
seed.
|
||||
- Keep max_tokens and max_model_len bounded for speed and memory use.
|
||||
|
||||
"""
|
||||
seed = int(os.getenv("VLLM_TEST_SEED", "12345"))
|
||||
random.seed(seed)
|
||||
@@ -395,8 +395,7 @@ def test_logprobs_bitwise_batch_invariance_bs1_vs_bsN(
|
||||
BACKENDS,
|
||||
)
|
||||
def test_simple_generation(backend):
|
||||
"""
|
||||
Simple test that runs the model with a basic prompt and prints the output.
|
||||
"""Simple test that runs the model with a basic prompt and prints the output.
|
||||
Useful for quick smoke testing and debugging.
|
||||
"""
|
||||
model = TEST_MODEL
|
||||
@@ -447,8 +446,7 @@ def test_simple_generation(backend):
|
||||
def test_logprobs_without_batch_invariance_should_fail(
|
||||
backend, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
"""
|
||||
This test is the inverse of test_logprobs_bitwise_batch_invariance_bs1_vs_bsN.
|
||||
"""This test is the inverse of test_logprobs_bitwise_batch_invariance_bs1_vs_bsN.
|
||||
It DISABLES batch invariance mode and expects to see non-deterministic behavior
|
||||
between BS=1 and BS=N runs. This demonstrates that batch invariance is actually
|
||||
doing something useful.
|
||||
@@ -660,8 +658,7 @@ def test_logprobs_without_batch_invariance_should_fail(
|
||||
def test_decode_logprobs_match_prefill_logprobs(
|
||||
backend,
|
||||
):
|
||||
"""
|
||||
Test that verifies decode logprobs match prefill logprobs.
|
||||
"""Test that verifies decode logprobs match prefill logprobs.
|
||||
|
||||
For each decoded token at position i:
|
||||
1. Run decode to generate N tokens and collect their logprobs
|
||||
@@ -927,8 +924,7 @@ def LLM_with_max_seqs(
|
||||
attention_config: dict | None = None,
|
||||
kernel_config: dict | None = None,
|
||||
) -> LLM:
|
||||
"""
|
||||
Helper to construct an LLM with a specific max_num_seqs (batch-size limit)
|
||||
"""Helper to construct an LLM with a specific max_num_seqs (batch-size limit)
|
||||
using the high-level v1 LLM API, while constraining memory usage.
|
||||
"""
|
||||
extra_kwargs: dict = {}
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Test batch-invariant matmul against torch.matmul for various shape combinations.
|
||||
"""Test batch-invariant matmul against torch.matmul for various shape combinations.
|
||||
|
||||
Tests correctness (matches torch.matmul) and batch invariance (result for one
|
||||
item doesn't change based on other items in the batch).
|
||||
@@ -53,9 +52,7 @@ DEVICE_TYPE = current_platform.device_type
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
def test_matmul_correctness(a_shape, b_shape, dtype):
|
||||
"""
|
||||
Compare matmul_batch_invariant against torch.matmul for various shapes.
|
||||
"""
|
||||
"""Compare matmul_batch_invariant against torch.matmul for various shapes."""
|
||||
device = torch.device(DEVICE_TYPE)
|
||||
|
||||
torch.manual_seed(42)
|
||||
@@ -87,11 +84,9 @@ def test_matmul_correctness(a_shape, b_shape, dtype):
|
||||
@skip_unsupported
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
def test_matmul_batch_invariance(dtype):
|
||||
"""
|
||||
Verify that the result for one item is bitwise identical regardless
|
||||
"""Verify that the result for one item is bitwise identical regardless
|
||||
of what other items are in the batch.
|
||||
"""
|
||||
|
||||
device = torch.device(DEVICE_TYPE)
|
||||
|
||||
torch.manual_seed(42)
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
HTTP-based batch invariance test: send requests to a running
|
||||
"""HTTP-based batch invariance test: send requests to a running
|
||||
vLLM server and compare BS=1 vs BS=N results (tokens and per-step logprobs).
|
||||
|
||||
Environment variables:
|
||||
|
||||
@@ -41,8 +41,7 @@ def test_rms_norm_batch_invariant_vs_reference(
|
||||
eps: float,
|
||||
seed: int,
|
||||
):
|
||||
"""
|
||||
Compare batch-invariant Triton RMS norm against a PyTorch reference.
|
||||
"""Compare batch-invariant Triton RMS norm against a PyTorch reference.
|
||||
|
||||
Tests that the Triton-based batch-invariant RMS norm produces numerically
|
||||
equivalent results to an independent implementation across configurations.
|
||||
@@ -90,9 +89,7 @@ def test_fused_add_rms_norm_batch_invariant_residual_path(
|
||||
n_extra: int,
|
||||
seed: int,
|
||||
):
|
||||
"""
|
||||
Test the batch-invariant fused residual-add + RMSNorm helper directly.
|
||||
"""
|
||||
"""Test the batch-invariant fused residual-add + RMSNorm helper directly."""
|
||||
device = torch.device(DEVICE_TYPE)
|
||||
|
||||
torch.manual_seed(seed)
|
||||
@@ -278,7 +275,8 @@ def test_rms_norm_per_block_quant_batch_invariant(
|
||||
hidden_size: int, dtype: torch.dtype, seed: int
|
||||
):
|
||||
"""C++ ``rms_norm_per_block_quant`` must be batch invariant across the
|
||||
block threshold (compiled fp8 block-quant path; block pinned to 512)."""
|
||||
block threshold (compiled fp8 block-quant path; block pinned to 512).
|
||||
"""
|
||||
import vllm._custom_ops as ops
|
||||
|
||||
device = torch.device(DEVICE_TYPE)
|
||||
@@ -313,8 +311,7 @@ def test_rms_norm_per_block_quant_batch_invariant(
|
||||
def test_rms_norm_3d_input(
|
||||
default_vllm_config, batch_size: int, seq_len: int, hidden_size: int
|
||||
):
|
||||
"""
|
||||
Test RMS norm with 3D input tensors (batch, seq_len, hidden_size).
|
||||
"""Test RMS norm with 3D input tensors (batch, seq_len, hidden_size).
|
||||
|
||||
Ensures that the batch-invariant RMS norm correctly handles multi-dimensional
|
||||
inputs that are common in transformer models.
|
||||
@@ -349,8 +346,7 @@ def test_rms_norm_3d_input(
|
||||
|
||||
@skip_if_not_cuda
|
||||
def test_rms_norm_numerical_stability(default_vllm_config):
|
||||
"""
|
||||
Test RMS norm numerical stability with extreme values.
|
||||
"""Test RMS norm numerical stability with extreme values.
|
||||
|
||||
Ensures that both implementations handle edge cases like very small or large
|
||||
values without producing NaN or Inf.
|
||||
@@ -406,8 +402,7 @@ def test_rms_norm_numerical_stability(default_vllm_config):
|
||||
|
||||
@skip_unsupported
|
||||
def test_rms_norm_formula(default_vllm_config):
|
||||
"""
|
||||
Test that RMS norm follows the correct mathematical formula.
|
||||
"""Test that RMS norm follows the correct mathematical formula.
|
||||
|
||||
Verifies: output = input / sqrt(mean(input^2) + eps) * weight
|
||||
"""
|
||||
@@ -440,8 +435,7 @@ def test_rms_norm_formula(default_vllm_config):
|
||||
@skip_if_not_cuda
|
||||
@pytest.mark.parametrize("hidden_size", [128, 1024, 4096, 16384])
|
||||
def test_rms_norm_different_hidden_sizes(default_vllm_config, hidden_size: int):
|
||||
"""
|
||||
Test RMS norm with various hidden sizes to ensure block size handling.
|
||||
"""Test RMS norm with various hidden sizes to ensure block size handling.
|
||||
|
||||
The Triton kernel uses a fixed BLOCK_SIZE=1024, so this tests that it
|
||||
correctly handles hidden sizes both smaller and larger than the block size.
|
||||
@@ -474,8 +468,7 @@ def test_rms_norm_different_hidden_sizes(default_vllm_config, hidden_size: int):
|
||||
|
||||
@skip_unsupported
|
||||
def test_rms_norm_determinism(default_vllm_config):
|
||||
"""
|
||||
Test that batch-invariant RMS norm produces deterministic results.
|
||||
"""Test that batch-invariant RMS norm produces deterministic results.
|
||||
|
||||
Runs the same input through the kernel multiple times and verifies
|
||||
identical outputs.
|
||||
|
||||
@@ -493,7 +493,6 @@ async def test_dp_pause_abort(expert_parallel: bool):
|
||||
@pytest.mark.parametrize("expert_parallel", [False, True])
|
||||
async def test_dp_pause_keep_then_resume(expert_parallel: bool):
|
||||
"""Start generation, pause after a few tokens (keep mode), resume; verify gap."""
|
||||
|
||||
pause_duration = 2.0
|
||||
min_tokens_before_pause = 3
|
||||
|
||||
@@ -548,7 +547,8 @@ async def test_dp_pause_keep_then_resume(expert_parallel: bool):
|
||||
async def test_dp_pause_keep_race_staggered_engines():
|
||||
"""Race: send pause(keep) to engine 0, then add two requests,
|
||||
then pause(keep) to engine 1. Ensures no deadlock when pause
|
||||
requests are staggered and requests arrive in between."""
|
||||
requests are staggered and requests arrive in between.
|
||||
"""
|
||||
if DP_SIZE != 2:
|
||||
pytest.skip("test_dp_pause_keep_race_staggered_engines requires DP_SIZE=2")
|
||||
|
||||
@@ -607,8 +607,7 @@ async def test_dp_pause_keep_race_staggered_engines():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dp_pause_barrier_request_deadlock():
|
||||
"""
|
||||
Test that start_dp_wave is ignored while paused.
|
||||
"""Test that start_dp_wave is ignored while paused.
|
||||
|
||||
Sequence:
|
||||
1. Pause all engines (PAUSED_ALL).
|
||||
@@ -719,7 +718,8 @@ async def test_dp_pause_barrier_request_deadlock():
|
||||
async def test_dp_pause_wait_mode_drains_in_flight():
|
||||
"""mode="wait" through the DP consensus path: the pause lets the
|
||||
in-flight request run to completion while the consensus is pending,
|
||||
resolves only once drained, and the engine generates again on resume."""
|
||||
resolves only once drained, and the engine generates again on resume.
|
||||
"""
|
||||
with ExitStack() as after:
|
||||
engine = AsyncLLM.from_engine_args(_get_dp_pause_engine_args(True))
|
||||
after.callback(engine.shutdown)
|
||||
@@ -750,7 +750,8 @@ async def test_dp_pause_wait_mode_drains_in_flight():
|
||||
async def test_dp_pause_while_asleep():
|
||||
"""Pausing a sleeping DP engine: sleeping ranks skip dummy batches yet
|
||||
must still reach the pause consensus, and the completion barrier must
|
||||
be harmless on workers whose memory is unmapped."""
|
||||
be harmless on workers whose memory is unmapped.
|
||||
"""
|
||||
with ExitStack() as after:
|
||||
engine = AsyncLLM.from_engine_args(_get_dp_pause_engine_args(True))
|
||||
after.callback(engine.shutdown)
|
||||
@@ -786,7 +787,8 @@ async def test_dp_pause_completion_implies_device_idle(
|
||||
"""A resolved pause future promises an idle device. Inflate the dummy
|
||||
batches the pause consensus manufactures so any work the pause fails to
|
||||
wait on stays visible, then require a quiet worker stream the moment
|
||||
pause_generation() returns."""
|
||||
pause_generation() returns.
|
||||
"""
|
||||
with ExitStack() as after:
|
||||
# The probes below ship functions through collective_rpc.
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Test Dual Batch Overlap (DBO) with Data Parallelism + Expert Parallelism.
|
||||
"""Test Dual Batch Overlap (DBO) with Data Parallelism + Expert Parallelism.
|
||||
|
||||
DBO is specifically designed for DP+EP scenarios to hide communication latency
|
||||
by overlapping computation of two batches. This test validates that DBO works
|
||||
@@ -54,9 +53,7 @@ DEEPEP_BACKENDS = [
|
||||
),
|
||||
)
|
||||
def test_dbo_dp_ep_gsm8k(all2all_backend: str, num_gpus_available):
|
||||
"""
|
||||
Test DBO with DP+EP using GSM8K evaluation.
|
||||
"""
|
||||
"""Test DBO with DP+EP using GSM8K evaluation."""
|
||||
lm_eval = pytest.importorskip("lm_eval")
|
||||
required_gpus = DP_SIZE
|
||||
|
||||
|
||||
@@ -24,7 +24,8 @@ TP_SIZE = int(os.getenv("TP_SIZE", "1"))
|
||||
|
||||
class ExternalLBServerManager:
|
||||
"""Manages data parallel vLLM server instances for external
|
||||
load balancer testing."""
|
||||
load balancer testing.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -30,7 +30,8 @@ DP_SIZE_LOCAL = DP_SIZE // NUM_NODES # 2 ranks per node
|
||||
class HybridLBServerManager:
|
||||
"""Manages hybrid data parallel vLLM server instances where each node
|
||||
runs a single logical API server that balances requests only to the
|
||||
DP engines running on that same node."""
|
||||
DP engines running on that same node.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -107,7 +107,8 @@ async def _run_request_bursts(
|
||||
|
||||
class MultinodeInternalLBServerManager:
|
||||
"""Manages multi-node data parallel vLLM server instances for internal
|
||||
load balancer testing using --headless mode."""
|
||||
load balancer testing using --headless mode.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -241,7 +242,8 @@ class MultinodeInternalLBServerManager:
|
||||
|
||||
class APIOnlyServerManager:
|
||||
"""Manages API-only server (Node 0) and headless engines server (Node 1)
|
||||
for testing separated API server and engine configuration."""
|
||||
for testing separated API server and engine configuration.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -261,7 +263,6 @@ class APIOnlyServerManager:
|
||||
|
||||
def __enter__(self) -> list[tuple[RemoteOpenAIServer, list[str]]]:
|
||||
"""Start API-only server and headless engines server."""
|
||||
|
||||
# Start API-only server (Node 0) - no engines, only API server
|
||||
api_server_args = self.base_server_args.copy()
|
||||
api_server_args.extend(
|
||||
@@ -607,7 +608,6 @@ async def test_api_only_multinode_dp_completion(
|
||||
model_name: str,
|
||||
) -> None:
|
||||
"""Test API-only server with all engines on separate headless server."""
|
||||
|
||||
# Test single request
|
||||
result = await _make_completion_request(api_only_client, model_name)
|
||||
assert result is not None
|
||||
@@ -645,7 +645,8 @@ async def test_api_only_multinode_dp_completion_streaming(
|
||||
model_name: str,
|
||||
) -> None:
|
||||
"""Test API-only server streaming with all engines on separate
|
||||
headless server."""
|
||||
headless server.
|
||||
"""
|
||||
prompt = "What is an LLM?"
|
||||
|
||||
async def make_streaming_request():
|
||||
|
||||
@@ -82,7 +82,8 @@ async def _generate(engine: AsyncLLM, prompt: str, max_tokens: int) -> int:
|
||||
@pytest.mark.parametrize("async_scheduling", [True, False])
|
||||
async def test_pp_dp_v2_low_concurrency(async_scheduling: bool):
|
||||
"""A single in-flight request at a time, repeated, to exercise the
|
||||
PP slot ring under empty batches between decodes."""
|
||||
PP slot ring under empty batches between decodes.
|
||||
"""
|
||||
with ExitStack() as after:
|
||||
engine = AsyncLLM.from_engine_args(_engine_args(async_scheduling))
|
||||
after.callback(engine.shutdown)
|
||||
@@ -96,7 +97,8 @@ async def test_pp_dp_v2_low_concurrency(async_scheduling: bool):
|
||||
@pytest.mark.parametrize("async_scheduling", [True, False])
|
||||
async def test_pp_dp_v2_mid_concurrency(async_scheduling: bool):
|
||||
"""64 concurrent requests, staggered, to exercise the steady-state
|
||||
DP all-reduce + PP slot-ring path."""
|
||||
DP all-reduce + PP slot-ring path.
|
||||
"""
|
||||
with ExitStack() as after:
|
||||
engine = AsyncLLM.from_engine_args(_engine_args(async_scheduling))
|
||||
after.callback(engine.shutdown)
|
||||
@@ -112,8 +114,8 @@ async def test_pp_dp_v2_mid_concurrency(async_scheduling: bool):
|
||||
@pytest.mark.asyncio
|
||||
async def test_pp_dp_v2_abort_mid_decode():
|
||||
"""Cancel half the in-flight requests mid-stream and confirm the
|
||||
engine survives the abort storm."""
|
||||
|
||||
engine survives the abort storm.
|
||||
"""
|
||||
with ExitStack() as after:
|
||||
engine = AsyncLLM.from_engine_args(_engine_args(async_scheduling=True))
|
||||
after.callback(engine.shutdown)
|
||||
@@ -153,8 +155,8 @@ async def test_pp_dp_v2_abort_mid_decode():
|
||||
@pytest.mark.asyncio
|
||||
async def test_pp_dp_v2_pause_resume():
|
||||
"""Pause an engine with a request in flight, then resume and confirm
|
||||
new requests still work."""
|
||||
|
||||
new requests still work.
|
||||
"""
|
||||
with ExitStack() as after:
|
||||
engine = AsyncLLM.from_engine_args(_engine_args(async_scheduling=True))
|
||||
after.callback(engine.shutdown)
|
||||
|
||||
@@ -48,7 +48,8 @@ def test_without_spec_decoding(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""Test consistency of combos of async scheduling, preemption,
|
||||
uni/multiproc executor, prefill chunking."""
|
||||
uni/multiproc executor, prefill chunking.
|
||||
"""
|
||||
struct_outputs = StructuredOutputsParams(json=sample_json_schema)
|
||||
test_sampling_params: list[dict[str, Any]] = [
|
||||
dict(),
|
||||
@@ -111,7 +112,6 @@ def test_with_eagle3_spec_decoding(sample_json_schema, monkeypatch: pytest.Monke
|
||||
preemption, executor, async scheduling, prefill chunking,
|
||||
spec decoding model length.
|
||||
"""
|
||||
|
||||
spec_config = {
|
||||
"method": "eagle3",
|
||||
"num_speculative_tokens": 2,
|
||||
@@ -171,7 +171,6 @@ def test_with_ngram_gpu_spec_decoding(monkeypatch: pytest.MonkeyPatch):
|
||||
- Async scheduling enabled (as in production)
|
||||
- Different executors and chunking settings
|
||||
"""
|
||||
|
||||
# Variant with larger speculation window
|
||||
ngram_gpu_config = {
|
||||
"method": "ngram_gpu",
|
||||
@@ -206,8 +205,8 @@ def run_tests(
|
||||
test_sampling_params: list[dict[str, Any]],
|
||||
):
|
||||
"""Test consistency of combos of async scheduling, preemption,
|
||||
uni/multiproc executor with spec decoding."""
|
||||
|
||||
uni/multiproc executor with spec decoding.
|
||||
"""
|
||||
# Flex attention supports float32.
|
||||
attention_config = {"backend": "FLEX_ATTENTION"}
|
||||
|
||||
|
||||
@@ -72,7 +72,8 @@ def test_backend_per_kind_splits_groups(backend_per_kind, monkeypatch):
|
||||
)
|
||||
def test_backend_per_kind_overrides_global_backend(monkeypatch):
|
||||
"""A per-kind entry wins over the global ``backend`` for its kind; other
|
||||
kinds fall back to the global backend."""
|
||||
kinds fall back to the global backend.
|
||||
"""
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
llm = LLM(
|
||||
model=MODEL,
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Tests for vLLM `vllm/v1/engine/processor.Processor._validate_model_input()`
|
||||
"""Tests for vLLM `vllm/v1/engine/processor.Processor._validate_model_input()`
|
||||
handling of maximum context length for decoder models.
|
||||
|
||||
This test ensures:
|
||||
@@ -35,8 +34,8 @@ def test_decoder_max_context_length_validation(
|
||||
max_tokens: int,
|
||||
) -> None:
|
||||
"""Check vLLM decoder model input validation for edge cases where
|
||||
the prompt length is (almost) equal to the max model length."""
|
||||
|
||||
the prompt length is (almost) equal to the max model length.
|
||||
"""
|
||||
prompt_ids = [[43] * prompt_len]
|
||||
|
||||
with vllm_runner(
|
||||
@@ -73,8 +72,8 @@ def test_auto_fit_max_model_len_rejects_oversized_input(
|
||||
"""When max_model_len='auto' and KV cache memory is very limited,
|
||||
the engine auto-fits max_model_len to a small value. The frontend
|
||||
must see this reduced value and reject prompts that exceed it,
|
||||
rather than accepting them and hanging."""
|
||||
|
||||
rather than accepting them and hanging.
|
||||
"""
|
||||
# Use a small KV cache budget to force auto-fit to a small
|
||||
# max_model_len. Pin block_size=16 so the budget is independent
|
||||
# of the platform's default block size. One block for this model is
|
||||
|
||||
@@ -35,8 +35,7 @@ model_config = {
|
||||
def test_sliding_window_retrieval(
|
||||
model, batch_size, seed, disable_hybrid_kv_cache_manager, vllm_runner
|
||||
):
|
||||
"""
|
||||
The test does a bunch of assignments "x1 = 10\nx2 = 33\n..." and then
|
||||
"""The test does a bunch of assignments "x1 = 10\nx2 = 33\n..." and then
|
||||
asks for value of one of them (which is outside the sliding window).
|
||||
If we tell it upfront which we are going to be looking for, then
|
||||
it answers correctly (mostly).
|
||||
@@ -112,14 +111,14 @@ def test_hybrid_kv_cache_manager_output_equivalence(model, vllm_runner):
|
||||
|
||||
|
||||
def check_length(prompts: list[str], llm: LLM, sliding_window: int):
|
||||
"""
|
||||
Check if the prompt length is valid, i.e., longer than the sliding window
|
||||
"""Check if the prompt length is valid, i.e., longer than the sliding window
|
||||
size and shorter than the model's max length.
|
||||
|
||||
Args:
|
||||
prompts: list of prompts
|
||||
llm: LLM object
|
||||
sliding_window: Sliding window size
|
||||
|
||||
"""
|
||||
tokenizer = llm.get_tokenizer()
|
||||
max_model_len = llm.llm_engine.model_config.max_model_len
|
||||
|
||||
@@ -17,9 +17,7 @@ SEED = 42
|
||||
|
||||
@pytest.fixture
|
||||
def test_prompts():
|
||||
"""
|
||||
Adapted from tests/v1/e2e/spec_decode/utils.py
|
||||
"""
|
||||
"""Adapted from tests/v1/e2e/spec_decode/utils.py."""
|
||||
prompt_types = ["repeat", "sentence"]
|
||||
# Setting higher num prompts increases the chance of numerics mismatch
|
||||
# due to matrix multiplication numerics depending on batch dimension
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Comprehensive end-to-end tests for `min_tokens` in the V1 engine.
|
||||
"""Comprehensive end-to-end tests for `min_tokens` in the V1 engine.
|
||||
|
||||
Addresses #21950: verify and add CI coverage.
|
||||
|
||||
@@ -28,7 +27,7 @@ GREEDY = 0.0 # Deterministic generation for consistent testing
|
||||
|
||||
|
||||
class MinTokensTestCase:
|
||||
"""Data class for min_tokens test scenarios"""
|
||||
"""Data class for min_tokens test scenarios."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -161,7 +160,7 @@ MIN_TOKENS_TEST_CASES = [
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def llm_v1():
|
||||
"""Create V1 LLM instance for testing"""
|
||||
"""Create V1 LLM instance for testing."""
|
||||
llm = LLM(
|
||||
model=TEST_MODEL,
|
||||
tensor_parallel_size=1,
|
||||
@@ -173,7 +172,8 @@ def llm_v1():
|
||||
|
||||
def test_min_tokens_with_terminal_structured_output(llm_v1: LLM):
|
||||
"""A grammar whose only legal continuation is EOS must stop under
|
||||
min_tokens instead of sampling outside the grammar."""
|
||||
min_tokens instead of sampling outside the grammar.
|
||||
"""
|
||||
params = SamplingParams(
|
||||
temperature=GREEDY,
|
||||
min_tokens=20,
|
||||
@@ -193,7 +193,7 @@ def test_min_tokens_with_terminal_structured_output(llm_v1: LLM):
|
||||
|
||||
|
||||
def get_token_count(output: RequestOutput) -> int:
|
||||
"""Extract token count from LLM output"""
|
||||
"""Extract token count from LLM output."""
|
||||
if not output.outputs:
|
||||
return 0
|
||||
return len(output.outputs[0].token_ids)
|
||||
@@ -202,7 +202,7 @@ def get_token_count(output: RequestOutput) -> int:
|
||||
def assert_min_tokens_satisfied(
|
||||
output: RequestOutput, test_case: MinTokensTestCase
|
||||
) -> None:
|
||||
"""Assert that min_tokens requirement is satisfied"""
|
||||
"""Assert that min_tokens requirement is satisfied."""
|
||||
token_count = get_token_count(output)
|
||||
stop_reason = output.outputs[0].stop_reason if output.outputs else "no output"
|
||||
|
||||
@@ -228,8 +228,7 @@ def assert_min_tokens_satisfied(
|
||||
ids=lambda tc: tc.name,
|
||||
)
|
||||
def test_min_tokens_comprehensive(llm_v1: LLM, test_case: MinTokensTestCase):
|
||||
"""
|
||||
Comprehensive test for min_tokens functionality in V1 engine.
|
||||
"""Comprehensive test for min_tokens functionality in V1 engine.
|
||||
|
||||
This test covers all critical scenarios for min_tokens:
|
||||
- Basic functionality (should work)
|
||||
@@ -240,6 +239,7 @@ def test_min_tokens_comprehensive(llm_v1: LLM, test_case: MinTokensTestCase):
|
||||
Args:
|
||||
llm_v1: V1 LLM instance
|
||||
test_case: Test scenario parameters
|
||||
|
||||
"""
|
||||
# Known failing cases are handled via param-level xfail marks above.
|
||||
|
||||
@@ -279,8 +279,7 @@ def test_min_tokens_comprehensive(llm_v1: LLM, test_case: MinTokensTestCase):
|
||||
|
||||
|
||||
def test_min_tokens_basic_functionality(llm_v1: LLM):
|
||||
"""
|
||||
Test basic min_tokens functionality without stop conditions.
|
||||
"""Test basic min_tokens functionality without stop conditions.
|
||||
|
||||
This is a baseline test that should always pass and validates
|
||||
that min_tokens works correctly in the simple case.
|
||||
@@ -302,8 +301,7 @@ def test_min_tokens_basic_functionality(llm_v1: LLM):
|
||||
strict=False,
|
||||
)
|
||||
def test_min_tokens_stop_strings_bug(llm_v1: LLM):
|
||||
"""
|
||||
Test the specific bug where stop strings bypass min_tokens.
|
||||
"""Test the specific bug where stop strings bypass min_tokens.
|
||||
|
||||
This test specifically reproduces the bug Calvin is fixing in PR #22014.
|
||||
It should fail until that fix is merged.
|
||||
@@ -353,8 +351,7 @@ def test_min_tokens_stop_strings_bug(llm_v1: LLM):
|
||||
strict=False,
|
||||
)
|
||||
def test_min_tokens_stop_strings_guaranteed_early_trigger(llm_v1: LLM):
|
||||
"""
|
||||
Guaranteed test for stop strings bypassing min_tokens bug.
|
||||
"""Guaranteed test for stop strings bypassing min_tokens bug.
|
||||
|
||||
Strategy: Use very low temperature and multiple common stop strings
|
||||
to virtually guarantee early detection, combined with long min_tokens
|
||||
@@ -408,8 +405,7 @@ def test_min_tokens_stop_strings_guaranteed_early_trigger(llm_v1: LLM):
|
||||
strict=False,
|
||||
)
|
||||
def test_min_tokens_eos_behavior(llm_v1: LLM):
|
||||
"""
|
||||
Verify EOS handling with and without min_tokens.
|
||||
"""Verify EOS handling with and without min_tokens.
|
||||
|
||||
- Without min_tokens: expect early EOS -> finish_reason == "stop",
|
||||
stop_reason is None, and generated tokens < max_tokens (25).
|
||||
@@ -492,8 +488,7 @@ def test_min_tokens_eos_behavior(llm_v1: LLM):
|
||||
|
||||
|
||||
def test_min_tokens_validation():
|
||||
"""
|
||||
Test that SamplingParams correctly validates min_tokens parameters.
|
||||
"""Test that SamplingParams correctly validates min_tokens parameters.
|
||||
|
||||
This tests the parameter validation logic in SamplingParams.
|
||||
"""
|
||||
|
||||
@@ -71,7 +71,6 @@ def retrieve_chunks(self):
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA not available")
|
||||
def test_pooling_chunked_prefill(vllm_runner, monkeypatch):
|
||||
"""Test chunked prefill for pooling models with LastPool."""
|
||||
|
||||
with monkeypatch.context() as m:
|
||||
m.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
model_id = "Qwen/Qwen3-Embedding-0.6B"
|
||||
@@ -124,7 +123,6 @@ def test_pooling_chunked_prefill(vllm_runner, monkeypatch):
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA not available")
|
||||
def test_pooling_prefix_cache(vllm_runner, monkeypatch):
|
||||
"""Test chunked prefill for pooling models with LastPool."""
|
||||
|
||||
verses = prompt.split("\n\n")
|
||||
|
||||
with monkeypatch.context() as m:
|
||||
|
||||
@@ -7,7 +7,8 @@ slightly different kernel/collective states (measured: same-state boots are
|
||||
bitwise identical across the two modes; cross-state boots differ by up to
|
||||
~0.25 logprob and can flip a near-tie token). A real sharding bug — rows
|
||||
routed to the wrong request, misaligned gathers — produces many-nats
|
||||
divergence and scrambled top-k sets, far outside these bounds."""
|
||||
divergence and scrambled top-k sets, far outside these bounds.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -106,7 +107,8 @@ def _assert_logprob_dicts_close(ref_lps, out_lps, what: str) -> None:
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
def test_sharded_sampling_outputs_match(monkeypatch: pytest.MonkeyPatch):
|
||||
"""Generation, logprobs, and prompt logprobs match between batch-sharded
|
||||
sampling and the replicated fallback, up to measured boot-state noise."""
|
||||
sampling and the replicated fallback, up to measured boot-state noise.
|
||||
"""
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
# Required for the collective_rpc mode check below.
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
End-to-end tests for the streaming input feature in AsyncLLM.
|
||||
"""End-to-end tests for the streaming input feature in AsyncLLM.
|
||||
|
||||
These tests verify that:
|
||||
1. Streaming inputs work correctly with bunched inputs (queued)
|
||||
|
||||
@@ -11,7 +11,8 @@ from ...utils import compute_acceptance_len, get_test_prompts
|
||||
@single_gpu_only
|
||||
def test_synthetic_acceptance_rate(vllm_runner):
|
||||
"""Verify that synthetic rejection sampling produces an acceptance
|
||||
length close to the requested mean acceptance length."""
|
||||
length close to the requested mean acceptance length.
|
||||
"""
|
||||
num_spec_tokens = 3
|
||||
expected_acceptance_len = 1.875
|
||||
tolerance = 0.15
|
||||
|
||||
@@ -43,8 +43,7 @@ def run_acceptance_length_eval(
|
||||
chat_template_kwargs: dict[str, Any],
|
||||
use_mrv2: bool,
|
||||
):
|
||||
"""
|
||||
E2E acceptance-rate validation for speculative decoding.
|
||||
"""E2E acceptance-rate validation for speculative decoding.
|
||||
|
||||
Drives one or more datasets (keyed in ``expected_acceptance_lengths``)
|
||||
through the spec decode engine and asserts the mean acceptance length
|
||||
|
||||
@@ -167,7 +167,8 @@ def test_draft_model_tensor_parallelism(vllm_runner):
|
||||
@multi_gpu_only(num_gpus=2)
|
||||
def test_draft_model_engine_args_tensor_parallelism():
|
||||
"""Ensure the vllm_config for the draft model is created correctly,
|
||||
and independently of the target model (quantization, TP, etc.)"""
|
||||
and independently of the target model (quantization, TP, etc.)
|
||||
"""
|
||||
engine_args = EngineArgs(
|
||||
model="Qwen/Qwen3-1.7B-FP8", # <<< tgt quantized
|
||||
tensor_parallel_size=2,
|
||||
@@ -198,7 +199,8 @@ def test_draft_model_engine_args_tensor_parallelism():
|
||||
|
||||
def _apply_draft_moe_backend(vllm_config: VllmConfig) -> VllmConfig:
|
||||
"""Replicate SpecDecodeBaseProposer._create_draft_vllm_config logic
|
||||
so we can test it without instantiating a full proposer."""
|
||||
so we can test it without instantiating a full proposer.
|
||||
"""
|
||||
spec_cfg = vllm_config.speculative_config
|
||||
if spec_cfg.moe_backend is not None:
|
||||
return replace(
|
||||
@@ -223,7 +225,8 @@ def _platform_moe_backend(monkeypatch: pytest.MonkeyPatch) -> MoEBackend:
|
||||
|
||||
def test_draft_model_moe_backend_override(monkeypatch: pytest.MonkeyPatch):
|
||||
"""When moe_backend is set in speculative_config, the draft VllmConfig
|
||||
should use it while the target keeps its own setting."""
|
||||
should use it while the target keeps its own setting.
|
||||
"""
|
||||
target_moe_backend = _platform_moe_backend(monkeypatch)
|
||||
engine_args = EngineArgs(
|
||||
model="Qwen/Qwen3-1.7B",
|
||||
@@ -248,7 +251,8 @@ def test_draft_model_moe_backend_override(monkeypatch: pytest.MonkeyPatch):
|
||||
|
||||
def test_draft_model_moe_backend_inherits_target(monkeypatch: pytest.MonkeyPatch):
|
||||
"""When moe_backend is not set in speculative_config, the draft should
|
||||
inherit the target's moe_backend."""
|
||||
inherit the target's moe_backend.
|
||||
"""
|
||||
target_moe_backend = _platform_moe_backend(monkeypatch)
|
||||
engine_args = EngineArgs(
|
||||
model="Qwen/Qwen3-1.7B",
|
||||
@@ -271,7 +275,8 @@ def test_draft_model_moe_backend_inherits_target(monkeypatch: pytest.MonkeyPatch
|
||||
|
||||
def test_draft_model_moe_backend_default_auto():
|
||||
"""When neither target nor draft set moe_backend explicitly, both should
|
||||
default to 'auto'."""
|
||||
default to 'auto'.
|
||||
"""
|
||||
engine_args = EngineArgs(
|
||||
model="Qwen/Qwen3-1.7B",
|
||||
tensor_parallel_size=1,
|
||||
@@ -292,8 +297,8 @@ def test_draft_model_moe_backend_default_auto():
|
||||
|
||||
def test_draft_model_engine_args_rejects_invalid_tp_argname():
|
||||
"""The user should pass "draft_tensor_parallel_size" rather than
|
||||
"tensor_parallel_size". We enforce this with validation."""
|
||||
|
||||
"tensor_parallel_size". We enforce this with validation.
|
||||
"""
|
||||
engine_args = EngineArgs(
|
||||
model="Qwen/Qwen3-1.7B",
|
||||
tensor_parallel_size=1,
|
||||
@@ -310,7 +315,8 @@ def test_draft_model_engine_args_rejects_invalid_tp_argname():
|
||||
|
||||
def assert_draft_model_correctness(args: ArgsTest, vllm_runner):
|
||||
"""Compare the outputs using and not using speculative decoding.
|
||||
In the greedy decoding case, the outputs must match EXACTLY."""
|
||||
In the greedy decoding case, the outputs must match EXACTLY.
|
||||
"""
|
||||
test_prompts: list[Messages] = get_messages(
|
||||
dataset=args.dataset, n=args.num_prompts
|
||||
)
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
This script contains:
|
||||
"""This script contains:
|
||||
1. test lora with speculative decoding for batch inference
|
||||
"""
|
||||
|
||||
@@ -59,8 +58,7 @@ def test_batch_inference_correctness(
|
||||
model_setup: tuple[str, str, str, str, int],
|
||||
vllm_runner,
|
||||
):
|
||||
"""
|
||||
Compare the outputs of a LLM with only Lora and a LLM with both SD and Lora.
|
||||
"""Compare the outputs of a LLM with only Lora and a LLM with both SD and Lora.
|
||||
Should be the same and no failure when doing batch inference.
|
||||
model_setup: (method, model_name, spec_model_name, lora_path, tp_size)
|
||||
"""
|
||||
|
||||
@@ -28,8 +28,7 @@ def _run_eagle_correctness(
|
||||
attn_backend: str,
|
||||
vllm_runner,
|
||||
):
|
||||
"""
|
||||
Compare the outputs of an original LLM and a speculative LLM
|
||||
"""Compare the outputs of an original LLM and a speculative LLM
|
||||
which should be the same when using eagle speculative decoding.
|
||||
"""
|
||||
method, model_name, spec_model_name, tp_size = model_setup
|
||||
|
||||
@@ -157,8 +157,7 @@ def test_mtp_correctness(
|
||||
expected_accuracy_threshold: float,
|
||||
vllm_runner,
|
||||
):
|
||||
"""
|
||||
Compare the outputs of a original LLM and a speculative LLM
|
||||
"""Compare the outputs of a original LLM and a speculative LLM
|
||||
which should be the same when using MTP speculative decoding. Due to some variance
|
||||
in the engine, it is possible for some outputs to differ, so we expect that at least
|
||||
6/10 output tokens match exactly, and that the GSM8k accuracy is above a precomputed
|
||||
|
||||
@@ -64,8 +64,7 @@ def test_ngram_gpu_default_with_async_scheduling(
|
||||
async_scheduling: bool,
|
||||
vllm_runner,
|
||||
):
|
||||
"""
|
||||
Test ngram_gpu speculative decoding (k=3) correctness with and without
|
||||
"""Test ngram_gpu speculative decoding (k=3) correctness with and without
|
||||
async scheduling, validated via GSM8K accuracy.
|
||||
Uses Qwen/Qwen3-8B (ref GSM8K accuracy: 87%-92%).
|
||||
"""
|
||||
@@ -99,8 +98,7 @@ def test_suffix_decoding_acceptance(
|
||||
model_name: str,
|
||||
vllm_runner,
|
||||
):
|
||||
"""
|
||||
Check that suffix decoding caching takes effect and improves acceptance
|
||||
"""Check that suffix decoding caching takes effect and improves acceptance
|
||||
lengths and acceptance rates over multiple runs of the same prompts.
|
||||
"""
|
||||
test_prompts = get_test_prompts(mm_enabled=False)
|
||||
|
||||
@@ -32,8 +32,7 @@ def test_speculators_model_integration(
|
||||
expected_accuracy_threshold: float,
|
||||
vllm_runner,
|
||||
):
|
||||
"""
|
||||
Test that speculators models work with the simplified integration.
|
||||
"""Test that speculators models work with the simplified integration.
|
||||
|
||||
This verifies the `vllm serve <speculator-model>` use case where
|
||||
speculative config is automatically detected from the model config
|
||||
|
||||
@@ -28,7 +28,8 @@ def test_mtp_sharded_sampling_equivalence(monkeypatch: pytest.MonkeyPatch):
|
||||
sampling under MTP spec decoding: the collectives move the same logits
|
||||
bytes, Gumbel keys derive from (request slot, position, seed), and slot
|
||||
assignment is rank-deterministic. Both runs here are spec decode with
|
||||
identical math, so outputs must match exactly."""
|
||||
identical math, so outputs must match exactly.
|
||||
"""
|
||||
tp_size = 2
|
||||
_skip_if_insufficient_gpus_for_tp(tp_size)
|
||||
model_name = "Qwen/Qwen3.5-0.8B-Base"
|
||||
|
||||
@@ -67,8 +67,8 @@ def test_mtp_speculative_mixed_batch_short_prefill(
|
||||
vllm_runner, model_name, enable_prefix_caching
|
||||
):
|
||||
"""Test to ensure MTP speculative decoding correctly handles
|
||||
short prefill chunks that fall below the reorder_batch_threshold."""
|
||||
|
||||
short prefill chunks that fall below the reorder_batch_threshold.
|
||||
"""
|
||||
# Set so large that both prefills will be classified as decodes in a mixed batch
|
||||
# note, with prefix caching we require chunk_size >= mamba_block_size
|
||||
chunk_size = 256 if not enable_prefix_caching else 2048
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
EPD Correctness Test
|
||||
"""EPD Correctness Test.
|
||||
|
||||
Tests that EPD (Encoder-Prefill-Decode) disaggregation produces the same
|
||||
outputs as a baseline single instance.
|
||||
@@ -104,6 +103,7 @@ def check_vllm_server(url: str, timeout=5, retries=10) -> bool:
|
||||
|
||||
Returns:
|
||||
True if the server is ready, False otherwise
|
||||
|
||||
"""
|
||||
for attempt in range(retries):
|
||||
try:
|
||||
@@ -138,6 +138,7 @@ def run_chat_completion(
|
||||
|
||||
Returns:
|
||||
Generated text content
|
||||
|
||||
"""
|
||||
with openai.OpenAI(
|
||||
api_key="EMPTY", base_url=base_url, timeout=120, max_retries=0
|
||||
|
||||
@@ -381,7 +381,8 @@ def test_pin_if_ready_none_when_absent():
|
||||
|
||||
def test_pin_if_ready_returns_unpinned_entry_when_not_ready():
|
||||
"""The producer needs "present but not ready" told apart from "absent":
|
||||
the first is NACKed as retryable, the second as a miss."""
|
||||
the first is NACKed as retryable, the second as a miss.
|
||||
"""
|
||||
cache = EmbeddingCache(num_blocks=4)
|
||||
cache.alloc("h", 1) # not-ready
|
||||
entry = cache.pin_if_ready("h")
|
||||
|
||||
@@ -131,7 +131,8 @@ def test_has_cache_item_false_when_not_consumer(monkeypatch):
|
||||
|
||||
def test_connector_keys_on_identifier_not_mm_hash(monkeypatch):
|
||||
"""The connector must key the encoder cache on feature.identifier (what
|
||||
has_cache_item is called with), NOT feature.mm_hash."""
|
||||
has_cache_item is called with), NOT feature.mm_hash.
|
||||
"""
|
||||
s = _make_scheduler(monkeypatch)
|
||||
req = _Request([_Feature("PROC_KEY", length=1, identifier="ENC_KEY")])
|
||||
s.update_state_after_alloc(req, 0)
|
||||
@@ -201,7 +202,8 @@ def test_loads_only_serves_hashes_touched_this_step(monkeypatch):
|
||||
|
||||
def test_repeated_reload_same_step_loads_once(monkeypatch):
|
||||
"""The same mm_hash requested twice in one step must appear in
|
||||
meta.loads exactly once."""
|
||||
meta.loads exactly once.
|
||||
"""
|
||||
s = _make_scheduler(monkeypatch)
|
||||
_seed_cached(s, "a", n_blocks=2)
|
||||
|
||||
@@ -224,7 +226,8 @@ def test_load_not_emitted_for_uncached_entry(monkeypatch):
|
||||
def test_load_pin_protects_blocks_until_completion(monkeypatch):
|
||||
"""Loaded blocks stay pinned until the worker reports the load memcpy
|
||||
complete. Observable: a save that needs eviction cannot reclaim a
|
||||
still-pinned loaded entry, but can once the completion report arrives."""
|
||||
still-pinned loaded entry, but can once the completion report arrives.
|
||||
"""
|
||||
# 2 blocks, both occupied by "a" (ready).
|
||||
s = _make_scheduler(monkeypatch, num_blocks=2)
|
||||
_seed_cached(s, "a", n_blocks=2)
|
||||
@@ -251,7 +254,8 @@ def test_load_pin_protects_blocks_until_completion(monkeypatch):
|
||||
|
||||
def _can_evict(s: ECCPUScheduler, mm_hash: str, n_blocks: int) -> bool:
|
||||
"""Whether a new save of `mm_hash` can claim `n_blocks`, i.e. whether the
|
||||
blocks currently held by other entries are reclaimable."""
|
||||
blocks currently held by other entries are reclaimable.
|
||||
"""
|
||||
s.update_state_after_alloc(_Request([_Feature(mm_hash, length=n_blocks)]), 0)
|
||||
meta = s.build_connector_meta(scheduler_output=None)
|
||||
return mm_hash in meta.saves
|
||||
@@ -311,7 +315,8 @@ def test_late_report_does_not_release_a_later_load_of_same_hash(monkeypatch):
|
||||
|
||||
def test_region_full_skips_save_and_never_blocks(monkeypatch):
|
||||
"""When the region is fully occupied by pinned entries, new saves are
|
||||
silently skipped and ensure_cache_available never blocks."""
|
||||
silently skipped and ensure_cache_available never blocks.
|
||||
"""
|
||||
s = _make_scheduler(monkeypatch, num_blocks=1)
|
||||
_seed_cached(s, "pinned", n_blocks=1)
|
||||
s._cache.pin("pinned")
|
||||
@@ -330,7 +335,8 @@ def test_region_full_skips_save_and_never_blocks(monkeypatch):
|
||||
|
||||
def test_producer_only_never_emits_loads(monkeypatch):
|
||||
"""A producer-only scheduler must never populate meta.loads, even when
|
||||
entries are ready."""
|
||||
entries are ready.
|
||||
"""
|
||||
s = _make_scheduler(monkeypatch, ec_role="ec_producer")
|
||||
req = _Request([_Feature("h1", length=1)])
|
||||
|
||||
@@ -372,7 +378,8 @@ def test_consumer_only_has_cache_item(monkeypatch):
|
||||
|
||||
def test_save_not_emitted_for_already_cached_entry(monkeypatch):
|
||||
"""An entry already in the cache (from a prior save) must not trigger
|
||||
a second allocation or appear in meta.saves again."""
|
||||
a second allocation or appear in meta.saves again.
|
||||
"""
|
||||
s = _make_scheduler(monkeypatch)
|
||||
req = _Request([_Feature("h1", length=1)])
|
||||
|
||||
@@ -393,7 +400,8 @@ def test_save_not_emitted_for_already_cached_entry(monkeypatch):
|
||||
|
||||
def test_update_connector_output_ignores_foreign_meta(monkeypatch):
|
||||
"""A worker output whose payload is not an ECCPUWorkerMetadata is a
|
||||
no-op — no cache mutation, no crash."""
|
||||
no-op — no cache mutation, no crash.
|
||||
"""
|
||||
s = _make_scheduler(monkeypatch)
|
||||
_seed_cached(s, "a", n_blocks=1)
|
||||
|
||||
@@ -407,7 +415,8 @@ def test_update_connector_output_ignores_foreign_meta(monkeypatch):
|
||||
|
||||
def test_update_connector_output_ignores_unknown_report(monkeypatch):
|
||||
"""Reports for a never-seen save hash or a transfer id this scheduler never
|
||||
dispatched are dropped rather than mutating unrelated state."""
|
||||
dispatched are dropped rather than mutating unrelated state.
|
||||
"""
|
||||
s = _make_scheduler(monkeypatch)
|
||||
s.update_connector_output(_WorkerOutput(saves=["gone"], loads=[4242]))
|
||||
assert s.has_cache_item("gone") is False
|
||||
@@ -419,7 +428,8 @@ def test_update_connector_output_ignores_unknown_report(monkeypatch):
|
||||
|
||||
def test_has_pending_push_work_tracks_inflight_save(monkeypatch):
|
||||
"""A dispatched-but-unconfirmed save keeps push work pending until the
|
||||
completion report marks it ready."""
|
||||
completion report marks it ready.
|
||||
"""
|
||||
s = _make_scheduler(monkeypatch)
|
||||
assert s.has_pending_push_work() is False
|
||||
|
||||
@@ -434,7 +444,8 @@ def test_has_pending_push_work_tracks_inflight_save(monkeypatch):
|
||||
|
||||
def test_has_pending_push_work_tracks_inflight_load(monkeypatch):
|
||||
"""A dispatched-but-unconfirmed load keeps push work pending until the
|
||||
unpin completion report arrives."""
|
||||
unpin completion report arrives.
|
||||
"""
|
||||
s = _make_scheduler(monkeypatch)
|
||||
_seed_cached(s, "a", n_blocks=1)
|
||||
assert s.has_pending_push_work() is False # ready + unpinned
|
||||
@@ -453,7 +464,8 @@ def test_has_pending_push_work_tracks_inflight_load(monkeypatch):
|
||||
|
||||
def test_shutdown_disables_roles_and_cleans_region(monkeypatch):
|
||||
"""After shutdown the scheduler serves no items and the region is
|
||||
cleaned up."""
|
||||
cleaned up.
|
||||
"""
|
||||
s = _make_scheduler(monkeypatch, num_blocks=2)
|
||||
_seed_cached(s, "a", n_blocks=2)
|
||||
assert s.has_cache_item("a") is True
|
||||
|
||||
@@ -98,7 +98,8 @@ def test_second_instance_opens_existing_file_and_shares_memory():
|
||||
def test_only_creator_unlinks_file_on_cleanup():
|
||||
"""Critical contract: if the non-creator unlinks, the creator's mmap path
|
||||
becomes a dangling backing file and a third opener would create a new one
|
||||
out from under the creator."""
|
||||
out from under the creator.
|
||||
"""
|
||||
instance_id = str(uuid.uuid4())
|
||||
r1 = ECSharedRegion(
|
||||
engine_id=instance_id,
|
||||
@@ -215,7 +216,8 @@ def test_madvise_einval_selects_fallback_for_whole_region(monkeypatch):
|
||||
|
||||
def test_non_creator_does_not_probe_or_populate(monkeypatch):
|
||||
"""Only the creator pre-faults. A second opener must not probe the advice
|
||||
nor re-touch pages the creator already populated."""
|
||||
nor re-touch pages the creator already populated.
|
||||
"""
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu import ec_shared_region as esr
|
||||
|
||||
instance_id = str(uuid.uuid4())
|
||||
@@ -312,7 +314,8 @@ def test_madvise_unexpected_oserror_propagates(monkeypatch):
|
||||
|
||||
def test_pin_memory_success_sets_flag(region):
|
||||
"""When cudaHostRegister returns 0, _is_pinned flips to True
|
||||
and cleanup will correspondingly call cudaHostUnregister."""
|
||||
and cleanup will correspondingly call cudaHostUnregister.
|
||||
"""
|
||||
fake_cudart = MagicMock()
|
||||
success = MagicMock()
|
||||
success.value = 0
|
||||
@@ -332,7 +335,8 @@ def test_pin_memory_success_sets_flag(region):
|
||||
|
||||
def test_pin_memory_failure_leaves_flag_false():
|
||||
"""If cudaHostRegister fails (non-zero), don't pretend it succeeded —
|
||||
cleanup must NOT call cudaHostUnregister on memory we never registered."""
|
||||
cleanup must NOT call cudaHostUnregister on memory we never registered.
|
||||
"""
|
||||
r = _make_region()
|
||||
try:
|
||||
fake_cudart = MagicMock()
|
||||
|
||||
@@ -363,7 +363,8 @@ def test_save_caches_coalesces_only_adjacent_blocks(
|
||||
|
||||
def test_save_caches_noop_when_mm_hash_not_in_saves(make_worker):
|
||||
"""When the scheduler hasn't pre-allocated blocks for ``mm_hash``,
|
||||
``save_caches`` + ``flush_saves`` is a pure no-op."""
|
||||
``save_caches`` + ``flush_saves`` is a pure no-op.
|
||||
"""
|
||||
worker = make_worker()
|
||||
sentinel = 0x42
|
||||
worker._region.blocks.fill_(sentinel)
|
||||
@@ -381,7 +382,8 @@ def test_save_caches_noop_when_mm_hash_not_in_saves(make_worker):
|
||||
)
|
||||
def test_save_caches_noop_for_non_save_rank(make_worker, tp_rank, pcp_rank):
|
||||
"""Only TP rank 0 + PCP rank 0 writes to mmap. All other combinations
|
||||
must skip the save entirely."""
|
||||
must skip the save entirely.
|
||||
"""
|
||||
worker = make_worker(tp_rank=tp_rank, pcp_rank=pcp_rank)
|
||||
sentinel = 0x42
|
||||
worker._region.blocks.fill_(sentinel)
|
||||
@@ -396,7 +398,8 @@ def test_save_caches_noop_for_non_save_rank(make_worker, tp_rank, pcp_rank):
|
||||
|
||||
def test_save_caches_raises_when_allocated_blocks_too_small(make_worker):
|
||||
"""``save_caches`` must raise ``AssertionError`` when the encoder output
|
||||
is larger than the allocated block space."""
|
||||
is larger than the allocated block space.
|
||||
"""
|
||||
worker = make_worker()
|
||||
src = torch.zeros(3 * _HIDDEN_DIM, dtype=_DTYPE)
|
||||
with pytest.raises(AssertionError, match="exceeds allocated blocks"):
|
||||
@@ -549,7 +552,8 @@ def test_start_load_caches_noop_when_loads_is_empty(make_worker):
|
||||
def test_start_load_caches_skips_cached_and_loads_new_in_same_step(make_worker):
|
||||
"""Every hash in ``meta.loads`` is copied from mmap, including one whose
|
||||
key is already resident in ``encoder_cache`` — there is no worker-side
|
||||
skip."""
|
||||
skip.
|
||||
"""
|
||||
worker = make_worker()
|
||||
n_blocks = 3
|
||||
src_orig = torch.arange(n_blocks * _HIDDEN_DIM, dtype=_DTYPE).reshape(
|
||||
@@ -778,7 +782,8 @@ def test_buffer_pool_is_reused_across_save_steps(make_worker):
|
||||
@_requires_accelerator
|
||||
def test_buffer_pool_is_reused_across_load_steps(make_worker):
|
||||
"""Once a load copy completes its descriptor buffers return to the pool and
|
||||
are reused by the next call — no reallocation."""
|
||||
are reused by the next call — no reallocation.
|
||||
"""
|
||||
worker = make_worker()
|
||||
worker._region.blocks[0].fill_(0x01)
|
||||
worker._region.blocks[1].fill_(0x02)
|
||||
@@ -849,7 +854,8 @@ def test_init_pins_memory_when_available(pin_available, expected_pinned):
|
||||
|
||||
def test_shutdown_calls_region_cleanup_and_swallows_errors(caplog_vllm):
|
||||
"""``shutdown`` must always call ``region.cleanup`` — and must never
|
||||
raise."""
|
||||
raise.
|
||||
"""
|
||||
worker = object.__new__(ECCPUWorker)
|
||||
mock_region = Mock(spec=ECSharedRegion)
|
||||
worker._region = mock_region
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
E2E test for the ECCPUConnector.
|
||||
"""E2E test for the ECCPUConnector.
|
||||
|
||||
Verifies:
|
||||
- Accuracy: outputs from EC CPU cache match fresh encoder computation.
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Unit tests for ECExampleConnector.
|
||||
"""
|
||||
"""Unit tests for ECExampleConnector."""
|
||||
|
||||
import os
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
@@ -43,7 +43,8 @@ def assign(proxy):
|
||||
|
||||
def _drive(assign, e_urls, counts):
|
||||
"""Feed a sequence of request item-counts through the cursor, as
|
||||
fanout_encoder_primer does one request at a time."""
|
||||
fanout_encoder_primer does one request at a time.
|
||||
"""
|
||||
cursor = 0
|
||||
all_urls = []
|
||||
for count in counts:
|
||||
|
||||
@@ -40,7 +40,8 @@ def test_metadata_saves_dict_operations():
|
||||
|
||||
def test_metadata_loads_carry_a_transfer_id_and_blocks():
|
||||
"""Each load entry is a (transfer_id, block_ids) pair, so the worker can
|
||||
report the transfer rather than the mm_hash."""
|
||||
report the transfer rather than the mm_hash.
|
||||
"""
|
||||
meta = ECCPUConnectorMetadata()
|
||||
|
||||
meta.loads["mm_hash_1"] = (0, [5, 6, 7])
|
||||
@@ -54,7 +55,7 @@ def test_metadata_loads_carry_a_transfer_id_and_blocks():
|
||||
|
||||
|
||||
def test_metadata_saves_and_loads_are_independent():
|
||||
"""saves and loads dicts are independent."""
|
||||
"""Saves and loads dicts are independent."""
|
||||
meta = ECCPUConnectorMetadata()
|
||||
|
||||
meta.saves["key"] = [1, 2]
|
||||
|
||||
@@ -79,7 +79,8 @@ def test_request_finished_announces_not_ready_entry(monkeypatch):
|
||||
"""A save whose GPU->mmap copy hasn't been confirmed complete yet is
|
||||
still announced: the entry can't be evicted before it's ready, and a read
|
||||
arriving too early is NACKed NACK_NOT_READY for the consumer to retry —
|
||||
protecting it is not this method's job."""
|
||||
protecting it is not this method's job.
|
||||
"""
|
||||
s = _sched_gate_off(monkeypatch)
|
||||
s._nixl_enabled = True
|
||||
s._peer_host, s._peer_port = "1.2.3.4", 5601
|
||||
|
||||
@@ -129,7 +129,8 @@ def test_producer_session_grant_nacks_missing_entry():
|
||||
|
||||
def test_producer_session_grant_nacks_not_ready_entry_as_retryable():
|
||||
"""An announced entry whose save is still in flight is distinguished from a
|
||||
miss, so the consumer retries instead of falling back to local compute."""
|
||||
miss, so the consumer retries instead of falling back to local compute.
|
||||
"""
|
||||
cache = EmbeddingCache(num_blocks=8)
|
||||
cache.alloc("h1", 2) # not ready
|
||||
s = _make_producer_session(cache)
|
||||
@@ -329,7 +330,6 @@ def test_consumer_xfer_quarantined_exception_settles():
|
||||
|
||||
def test_consumer_xfer_quarantined_timeout_evicts_blocks():
|
||||
"""Quarantine timeout should release handle and return READ_FAILED."""
|
||||
|
||||
data = _make_data(xfer_state="PROC")
|
||||
x = _started(data)
|
||||
# First poll transitions to QUARANTINED (deadline extended by 60s)
|
||||
@@ -438,7 +438,8 @@ def test_consumer_session_poll_nack_goes_to_tombstoned():
|
||||
|
||||
def test_consumer_session_poll_not_ready_nack_goes_to_retryable():
|
||||
"""A not-ready NACK must not tombstone: tombstoning admits the request with
|
||||
no embedding, which is fatal once the media has been rewritten away."""
|
||||
no embedding, which is fatal once the media has been rewritten away.
|
||||
"""
|
||||
import msgspec
|
||||
|
||||
s = _make_consumer_session()
|
||||
|
||||
@@ -45,6 +45,7 @@ def create_ec_vllm_config(
|
||||
|
||||
Returns:
|
||||
A `VllmConfig`-specced mock carrying the configs above.
|
||||
|
||||
"""
|
||||
parallel_config = ParallelConfig(
|
||||
tensor_parallel_size=tensor_parallel_size,
|
||||
|
||||
@@ -24,12 +24,12 @@ EngineCorePromptLogprobsType = tuple[torch.Tensor, torch.Tensor]
|
||||
|
||||
|
||||
def _build_test_vectors_no_logprobs() -> DummyOutputProcessorTestVectors:
|
||||
"""Generate output processor dummy test vectors, without logprobs
|
||||
"""Generate output processor dummy test vectors, without logprobs.
|
||||
|
||||
Returns:
|
||||
DummyOutputProcessorTestVectors instance with no logprobs
|
||||
"""
|
||||
|
||||
"""
|
||||
tokenizer = AutoTokenizer.from_pretrained(TOKENIZER_NAME)
|
||||
vllm_config = EngineArgs(model=TOKENIZER_NAME).create_engine_config()
|
||||
# Tokenize prompts under test & create dummy generated tokens
|
||||
@@ -62,10 +62,11 @@ def _build_test_vectors_no_logprobs() -> DummyOutputProcessorTestVectors:
|
||||
|
||||
@pytest.fixture
|
||||
def dummy_test_vectors() -> DummyOutputProcessorTestVectors:
|
||||
"""Generate output processor dummy test vectors, with logprobs
|
||||
"""Generate output processor dummy test vectors, with logprobs.
|
||||
|
||||
Returns:
|
||||
DummyOutputProcessorTestVectors instance with logprobs
|
||||
|
||||
"""
|
||||
# Build dummy test vectors without logprobs
|
||||
dtv = _build_test_vectors_no_logprobs()
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
Test for the fix in PR #29987: Eagerly abort cancelled final-step requests.
|
||||
"""Test for the fix in PR #29987: Eagerly abort cancelled final-step requests.
|
||||
|
||||
This test verifies that when a request is aborted during its final execution
|
||||
step (when it would naturally complete), it is properly marked as aborted
|
||||
@@ -52,8 +51,7 @@ class DummyKVConnectorMetadata(KVConnectorMetadata):
|
||||
|
||||
|
||||
class DummyKVConnector(KVConnectorBase_V1):
|
||||
"""
|
||||
Dummy KV connector that captures request finish statuses to a file.
|
||||
"""Dummy KV connector that captures request finish statuses to a file.
|
||||
This is used to verify the fix - without the fix, a request aborted
|
||||
during its final step would be captured as FINISHED_LENGTH_CAPPED
|
||||
instead of FINISHED_ABORTED.
|
||||
@@ -144,8 +142,7 @@ KVConnectorFactory.register_connector(
|
||||
@pytest.mark.parametrize("async_scheduling", [False, True])
|
||||
@pytest.mark.asyncio
|
||||
async def test_abort_during_final_step(async_scheduling: bool):
|
||||
"""
|
||||
Test that a request aborted during its final execution step is treated as
|
||||
"""Test that a request aborted during its final execution step is treated as
|
||||
aborted rather than completed.
|
||||
|
||||
This test:
|
||||
@@ -165,7 +162,6 @@ async def test_abort_during_final_step(async_scheduling: bool):
|
||||
With the fix, _process_aborts_queue() runs before update_from_output(), so the
|
||||
abort takes precedence and the KV connector sees FINISHED_ABORTED.
|
||||
"""
|
||||
|
||||
# Create three temporary files:
|
||||
# 1. ready_file: deleted by execute_model to signal it has started
|
||||
# 2. block_file: execute_model waits for this to be deleted
|
||||
|
||||
@@ -457,7 +457,6 @@ async def test_customize_loggers(monkeypatch):
|
||||
If a customized logger is provided at the init, it should
|
||||
be added to the default loggers.
|
||||
"""
|
||||
|
||||
with ExitStack() as after:
|
||||
with set_default_torch_num_threads(1):
|
||||
engine = AsyncLLM.from_engine_args(
|
||||
@@ -642,7 +641,6 @@ async def test_check_health():
|
||||
@pytest.mark.asyncio
|
||||
async def test_abort_final_output(output_kind: RequestOutputKind):
|
||||
"""Test that abort() returns a final output with correct information."""
|
||||
|
||||
with ExitStack() as after:
|
||||
with set_default_torch_num_threads(1):
|
||||
engine = AsyncLLM.from_engine_args(TEXT_ENGINE_ARGS)
|
||||
|
||||
@@ -68,7 +68,8 @@ class _StubEngineCoreActor(EngineCoreActorMixin):
|
||||
# in ``tests/entrypoints/test_api_server_process_manager.py``.
|
||||
def _bind_and_report_worker(listen_address, sock, args, client_config):
|
||||
"""Bind ROUTER/PULL with a kernel-assigned port, report the actual
|
||||
endpoints back via ``actual_address_pipe``, then exit."""
|
||||
endpoints back via ``actual_address_pipe``, then exit.
|
||||
"""
|
||||
ctx = zmq.Context()
|
||||
try:
|
||||
in_sock = make_zmq_socket(
|
||||
|
||||
@@ -193,8 +193,7 @@ def test_engine_core():
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
def test_engine_core_advanced_sampling():
|
||||
"""
|
||||
A basic end-to-end test to verify that the engine functions correctly
|
||||
"""A basic end-to-end test to verify that the engine functions correctly
|
||||
when additional sampling parameters, such as top_p, min_tokens, and
|
||||
presence_penalty, are set.
|
||||
"""
|
||||
@@ -242,9 +241,7 @@ def test_engine_core_advanced_sampling():
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
def test_engine_core_concurrent_batches():
|
||||
"""
|
||||
Test that the engine can handle multiple concurrent batches.
|
||||
"""
|
||||
"""Test that the engine can handle multiple concurrent batches."""
|
||||
|
||||
def make_request_with_max_tokens(req_id: str, max_tokens: int) -> EngineCoreRequest:
|
||||
request = make_request()
|
||||
@@ -265,7 +262,6 @@ def test_engine_core_concurrent_batches():
|
||||
non_block=False,
|
||||
) -> Future[ModelRunnerOutput | None]:
|
||||
"""Make execute_model non-blocking."""
|
||||
|
||||
# DummyExecutor used only for testing async case.
|
||||
assert non_block
|
||||
|
||||
@@ -282,7 +278,6 @@ def test_engine_core_concurrent_batches():
|
||||
self, grammar_output, non_block=False
|
||||
) -> Future[ModelRunnerOutput]:
|
||||
"""Make sample_tokens non-blocking."""
|
||||
|
||||
# DummyExecutor used only for testing async case.
|
||||
assert non_block
|
||||
|
||||
@@ -401,10 +396,7 @@ def test_engine_core_concurrent_batches():
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
def test_engine_core_tp():
|
||||
"""
|
||||
Test engine can initialize worker in tp properly
|
||||
"""
|
||||
|
||||
"""Test engine can initialize worker in tp properly."""
|
||||
"""Setup the EngineCore."""
|
||||
engine_args = EngineArgs(
|
||||
model=MODEL_NAME,
|
||||
@@ -490,7 +482,7 @@ def test_encoder_instance_zero_kv_cache(
|
||||
enable_prefix_caching: bool,
|
||||
use_kv_connector: bool,
|
||||
):
|
||||
"""EPD (Encoder-Prefill-Decode) Encoder-cache-specific tests
|
||||
"""EPD (Encoder-Prefill-Decode) Encoder-cache-specific tests.
|
||||
|
||||
This test verifies encoder-only instance initializes with 0 KV cache blocks.
|
||||
Under EPD disagg mode, Encoder instances (EC producer role) only execute
|
||||
@@ -615,7 +607,8 @@ def _cadenced_dp_engine_core(
|
||||
monkeypatch, results: list[tuple[bool, bool]], dp_sync_interval: int = 32
|
||||
):
|
||||
"""A bare DPEngineCoreProc whose all-reduce is scripted by `results`;
|
||||
returns the core and the step numbers at which the all-reduce ran."""
|
||||
returns the core and the step numbers at which the all-reduce ran.
|
||||
"""
|
||||
core = object.__new__(DPEngineCoreProc)
|
||||
core.dp_group = object()
|
||||
core.dp_sync_interval = dp_sync_interval
|
||||
@@ -634,7 +627,8 @@ def _cadenced_dp_engine_core(
|
||||
|
||||
def test_dp_sync_interval_default_is_16():
|
||||
"""Regression pin: lowering the default narrows the mid-wave pause tail
|
||||
for async RL, where the engine is rarely idle when pause lands."""
|
||||
for async RL, where the engine is rarely idle when pause lands.
|
||||
"""
|
||||
assert ParallelConfig.dp_sync_interval == 16
|
||||
|
||||
|
||||
@@ -652,7 +646,8 @@ def test_dp_sync_interval_normal_wave(monkeypatch, dp_sync_interval: int):
|
||||
def test_dp_sync_interval_idle_pause_consensus_on_first_step(monkeypatch):
|
||||
"""A pause of an idle engine arms every rank before its kick-started
|
||||
first step, so the step-1 sync reaches consensus after one dummy batch
|
||||
regardless of the configured cadence."""
|
||||
regardless of the configured cadence.
|
||||
"""
|
||||
core, synced = _cadenced_dp_engine_core(
|
||||
monkeypatch, [(False, True)], dp_sync_interval=32
|
||||
)
|
||||
@@ -678,7 +673,8 @@ def _pausable_engine_core_proc() -> EngineCoreProc:
|
||||
@pytest.mark.parametrize("deferred", [False, True])
|
||||
def test_pause_synchronizes_device_before_cache_reset(deferred: bool):
|
||||
"""A resolved pause promises an idle device: the barrier must run before
|
||||
caches are cleared and before the caller is unblocked."""
|
||||
caches are cleared and before the caller is unblocked.
|
||||
"""
|
||||
core = _pausable_engine_core_proc()
|
||||
core.engines_running = deferred
|
||||
order: list[str] = []
|
||||
|
||||
@@ -249,7 +249,8 @@ def test_dplb_non_late_interaction_still_uses_lb():
|
||||
|
||||
def test_dplb_burst_round_robins_despite_snapshot_rebinds():
|
||||
"""A stats snapshot rebind wipes the optimistic lb_engines increments;
|
||||
the exact in-flight floor must keep a burst spreading round-robin."""
|
||||
the exact in-flight floor must keep a burst spreading round-robin.
|
||||
"""
|
||||
client = _make_dplb_client(num_engines=4)
|
||||
|
||||
for _ in range(4):
|
||||
@@ -324,7 +325,8 @@ async def test_dplb_scale_down_routes_after_stale_stats_snapshot():
|
||||
|
||||
def test_dplb_snapshot_backpressure_overrides_inflight():
|
||||
"""An engine reported heavily loaded by the coordinator is avoided even
|
||||
when this client has routed nothing to it."""
|
||||
when this client has routed nothing to it.
|
||||
"""
|
||||
client = _make_dplb_client(num_engines=2)
|
||||
client.lb_engines = [[5, 10, 0.0], [0, 0, 0.0]]
|
||||
|
||||
@@ -337,7 +339,8 @@ def test_dplb_snapshot_backpressure_overrides_inflight():
|
||||
|
||||
def test_dplb_kv_pressure_amplifies_waiting_penalty():
|
||||
"""A waiting queue on a KV-bound engine (slow drain) is penalized, while
|
||||
the same queue with low KV usage is not (e.g. transient burst)."""
|
||||
the same queue with low KV usage is not (e.g. transient burst).
|
||||
"""
|
||||
client = _make_dplb_client(num_engines=2)
|
||||
# Engine 0 has a smaller total but is KV-bound with a queue.
|
||||
client.lb_engines = [[5, 10, 1.0], [0, 20, 0.2]]
|
||||
@@ -1392,11 +1395,9 @@ def test_startup_failure(monkeypatch: pytest.MonkeyPatch):
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
def test_engine_core_proc_instantiation_cuda_empty(monkeypatch: pytest.MonkeyPatch):
|
||||
"""
|
||||
Test that EngineCoreProc can be instantiated when CUDA_VISIBLE_DEVICES
|
||||
"""Test that EngineCoreProc can be instantiated when CUDA_VISIBLE_DEVICES
|
||||
is empty. This ensures the engine frontend does not need access to GPUs.
|
||||
"""
|
||||
|
||||
from vllm.v1.engine.core import EngineCoreProc
|
||||
from vllm.v1.executor.abstract import Executor
|
||||
|
||||
|
||||
@@ -11,8 +11,7 @@ from vllm.v1.engine.detokenizer import IncrementalDetokenizer
|
||||
|
||||
|
||||
def test_fast_inc_detok_invalid_utf8_err_case():
|
||||
"""
|
||||
Test edge case where tokenizer can produce non-monotonic,
|
||||
"""Test edge case where tokenizer can produce non-monotonic,
|
||||
invalid UTF-8 output, which breaks the internal state of
|
||||
tokenizers' DecodeStream.
|
||||
See https://github.com/vllm-project/vllm/issues/17448.
|
||||
|
||||
@@ -82,7 +82,7 @@ def _get_test_sampling_params(
|
||||
rng = random.Random(seed)
|
||||
|
||||
def get_mostly_n_gt1() -> int:
|
||||
r"""Mostly n \in [2,20], ~1/3 n=1"""
|
||||
r"""Mostly n \in [2,20], ~1/3 n=1."""
|
||||
x = rng.randint(0, 28)
|
||||
if x < 10:
|
||||
return 1
|
||||
@@ -125,6 +125,7 @@ def test_parallel_sampling(vllm_model, example_prompts) -> None:
|
||||
Args:
|
||||
vllm_model: VllmRunner instance under test.
|
||||
example_prompt: test fixture providing prompts for testing.
|
||||
|
||||
"""
|
||||
sampling_params_list, n_list = _get_test_sampling_params(example_prompts)
|
||||
llm: LLM = vllm_model.llm
|
||||
|
||||
@@ -30,7 +30,8 @@ def _make_processor(num_logprobs: int) -> LogprobsProcessor:
|
||||
|
||||
def test_drops_trailing_sentinel_columns():
|
||||
"""A request that asked for 3 custom token logprobs but ended up in a
|
||||
batch padded to width 5 must not surface the trailing -inf entries."""
|
||||
batch padded to width 5 must not surface the trailing -inf entries.
|
||||
"""
|
||||
processor = _make_processor(num_logprobs=3)
|
||||
|
||||
sampled = 42
|
||||
|
||||
@@ -73,6 +73,7 @@ def _ref_convert_id_to_token(
|
||||
|
||||
Returns:
|
||||
String representation of input token id
|
||||
|
||||
"""
|
||||
return tokenizer.decode([token_id]) or ""
|
||||
|
||||
@@ -228,7 +229,8 @@ def test_request_stream_interval_raises_but_not_below_engine_default(
|
||||
):
|
||||
"""A per-request stream_interval can raise the interval above the engine
|
||||
default but not below it (values under the default clamp up), without
|
||||
altering the generated text."""
|
||||
altering the generated text.
|
||||
"""
|
||||
engine_stream_interval = 5
|
||||
# Request 0 (below the default) clamps up to 5; request 1 raises it to 10.
|
||||
request_stream_intervals = [1, 10]
|
||||
@@ -792,6 +794,7 @@ def test_stop_token(
|
||||
stop_token_type: "eos_token_id" for EOS, "stop_token_ids" for stop token
|
||||
ignore_eos: if True, EOS stops are disabled
|
||||
dummy_test_vectors: dummy engine core outputs and other data structures
|
||||
|
||||
"""
|
||||
model_id = dummy_test_vectors.tokenizer.name_or_path
|
||||
if model_id != "meta-llama/Llama-3.2-1B":
|
||||
|
||||
@@ -14,7 +14,6 @@ MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
|
||||
|
||||
def test_preprocess_error_handling(monkeypatch: pytest.MonkeyPatch):
|
||||
"""Test that preprocessing errors are handled gracefully."""
|
||||
|
||||
if current_platform.is_rocm() or current_platform.is_xpu():
|
||||
pytest.skip(
|
||||
"Skipped on ROCm/XPU: this test only works with 'fork', "
|
||||
|
||||
@@ -54,6 +54,7 @@ def _create_random_top_logprob_test_vector(
|
||||
|
||||
Returns:
|
||||
1D length-`num_logprobs` torch Tensor of float logprob values
|
||||
|
||||
"""
|
||||
return torch.rand(num_logprobs) * (upper - lower) + lower
|
||||
|
||||
@@ -79,6 +80,7 @@ def _create_random_top_logprob_test_matrix(
|
||||
|
||||
Returns:
|
||||
2D num_tokens x num_logprobs torch Tensor of float logprob values
|
||||
|
||||
"""
|
||||
return torch.rand(*shape) * (upper - lower) + lower
|
||||
|
||||
@@ -90,7 +92,7 @@ def _create_random_top_token_test_vector(
|
||||
sampled_token_id: int,
|
||||
adjust_num_logprobs: bool = True,
|
||||
) -> tuple[torch.Tensor, int]:
|
||||
"""Create a random vector of top logprob token indices
|
||||
"""Create a random vector of top logprob token indices.
|
||||
|
||||
Use to create fake sample logprobs for testing. The sampled token
|
||||
ID must always be one of the top logprobs, which this dummy test
|
||||
@@ -117,8 +119,8 @@ def _create_random_top_token_test_vector(
|
||||
sampled_token_rank: the rank of sampled_token_id in the vocab
|
||||
vector when sorted in descending order by
|
||||
logprob
|
||||
"""
|
||||
|
||||
"""
|
||||
# Calculate the final number of logprobs required
|
||||
total_logprobs = num_logprobs + 1 if adjust_num_logprobs else num_logprobs
|
||||
|
||||
@@ -148,7 +150,7 @@ def _create_random_top_token_test_matrix(
|
||||
upper: int,
|
||||
tokens_list: list[int],
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Create a random matrix of top logprob token indices
|
||||
"""Create a random matrix of top logprob token indices.
|
||||
|
||||
Use to create fake prompt logprobs for testing.
|
||||
|
||||
@@ -166,6 +168,7 @@ def _create_random_top_token_test_matrix(
|
||||
- 2D num_tokens x num_logprobs+1 torch Tensor of token ids
|
||||
- 1D tensor of ranks of prompt tokens in their respective
|
||||
rows, or random values
|
||||
|
||||
"""
|
||||
num_elements = shape[0] * shape[1]
|
||||
choice_tensor = torch.randperm(upper - lower)[:num_elements] + lower
|
||||
@@ -205,6 +208,7 @@ def decode_token(
|
||||
|
||||
Returns:
|
||||
string representation of token
|
||||
|
||||
"""
|
||||
return tokenizer.convert_ids_to_tokens(tok_id)
|
||||
|
||||
@@ -214,7 +218,7 @@ def generate_dummy_sample_logprobs(
|
||||
num_logprobs: int,
|
||||
tokenizer: PythonBackend,
|
||||
) -> list[tuple[list[int], list[float], int]]:
|
||||
"""Generate dummy sample logprobs
|
||||
"""Generate dummy sample logprobs.
|
||||
|
||||
Generate a test data structure which imitates the list of sample logprobs
|
||||
which would be assembled in the engine core during decode phase.
|
||||
@@ -224,13 +228,14 @@ def generate_dummy_sample_logprobs(
|
||||
num_logprobs: return `num_logprobs` or `num_logprobs+1` logprobs per token
|
||||
tokenizer: model tokenizer to use for detokenization
|
||||
|
||||
Returns
|
||||
Returns:
|
||||
list of (top token ids vector, logprobs vector, sampled token rank)
|
||||
Python lists tuples; in each tuple the logprobs and top token ids
|
||||
vectors have the same length which is either `num_logprobs` or
|
||||
`num_logprobs+1`. Sampled token rank is the rank (index+1) of the
|
||||
sampled token within the vocab vector when sorted by logprob in
|
||||
descending order.
|
||||
|
||||
"""
|
||||
res = []
|
||||
for sampled_token_id in sampled_tokens_list:
|
||||
@@ -263,7 +268,7 @@ def generate_dummy_prompt_logprobs_tensors(
|
||||
num_logprobs: int,
|
||||
tokenizer: PythonBackend,
|
||||
) -> LogprobsTensors:
|
||||
"""Generate dummy prompt logprobs tensors
|
||||
"""Generate dummy prompt logprobs tensors.
|
||||
|
||||
Generate a test data structure which imitates the torch Tensors of prompt
|
||||
logprobs which would be assembled in the engine core during chunked
|
||||
@@ -274,10 +279,11 @@ def generate_dummy_prompt_logprobs_tensors(
|
||||
num_logprobs: return `num_logprobs` logprobs per token
|
||||
tokenizer: model tokenizer to use for detokenization
|
||||
|
||||
Returns
|
||||
Returns:
|
||||
Single tuple of (logprobs matrix, top token ids matrix) torch Tensor,
|
||||
where both matrices have dimensions
|
||||
num_prompt_tokens x num_logprobs
|
||||
|
||||
"""
|
||||
# For now, assume the whole prompt is processed in one chunk; thus,
|
||||
# the number of non-`None` prompt logprobs is `len(prompt_tokens_list)-1`.
|
||||
@@ -308,7 +314,7 @@ def generate_dummy_prompt_logprobs_tensors(
|
||||
|
||||
@dataclass
|
||||
class DummyOutputProcessorTestVectors:
|
||||
"""Dummy test vectors for output processor tests"""
|
||||
"""Dummy test vectors for output processor tests."""
|
||||
|
||||
tokenizer: GeneralTokenizerType
|
||||
vllm_config: EngineArgs
|
||||
|
||||
@@ -59,6 +59,7 @@ class PredictableLlamaModel(nn.Module, EagleModelMixin):
|
||||
Returns:
|
||||
If aux_hidden_state_layers is set: (hidden_states, aux_hidden_states)
|
||||
Otherwise: hidden_states
|
||||
|
||||
"""
|
||||
# Determine sequence length
|
||||
if inputs_embeds is not None:
|
||||
|
||||
@@ -25,8 +25,7 @@ SAMPLE_PROMPTS = (
|
||||
|
||||
|
||||
def check_vllm_server(url: str, timeout=5, retries=3) -> bool:
|
||||
"""
|
||||
Checks if the vLLM server is ready by sending a GET request to the
|
||||
"""Checks if the vLLM server is ready by sending a GET request to the
|
||||
/health endpoint.
|
||||
|
||||
Args:
|
||||
@@ -36,6 +35,7 @@ def check_vllm_server(url: str, timeout=5, retries=3) -> bool:
|
||||
|
||||
Returns:
|
||||
bool: True if the server is ready, False otherwise.
|
||||
|
||||
"""
|
||||
for attempt in range(retries):
|
||||
try:
|
||||
@@ -81,8 +81,7 @@ def run_simple_prompt(
|
||||
|
||||
|
||||
def main():
|
||||
"""
|
||||
This script demonstrates how to accept two optional string arguments
|
||||
"""This script demonstrates how to accept two optional string arguments
|
||||
("service_url" and "file_name") from the command line, each with a
|
||||
default value of an empty string, using the argparse module.
|
||||
"""
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Integration edge-case tests for MultiConnector (NixlConnector + OffloadingConnector).
|
||||
"""Integration edge-case tests for MultiConnector (NixlConnector + OffloadingConnector).
|
||||
|
||||
Tests cover:
|
||||
- Output correctness across block-size boundaries (proxy vs direct prefill).
|
||||
|
||||
@@ -18,9 +18,7 @@ logger.setLevel(logging.DEBUG)
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
"""
|
||||
Lifespan context manager to handle startup and shutdown events.
|
||||
"""
|
||||
"""Lifespan context manager to handle startup and shutdown events."""
|
||||
# Startup: Initialize client pools for prefiller and decoder services
|
||||
app.state.prefill_clients = []
|
||||
app.state.decode_clients = []
|
||||
@@ -132,8 +130,7 @@ def parse_args():
|
||||
|
||||
|
||||
def get_next_client(app, service_type: str):
|
||||
"""
|
||||
Get the next client in round-robin fashion.
|
||||
"""Get the next client in round-robin fashion.
|
||||
|
||||
Args:
|
||||
app: The FastAPI app instance
|
||||
@@ -141,6 +138,7 @@ def get_next_client(app, service_type: str):
|
||||
|
||||
Returns:
|
||||
The next client to use
|
||||
|
||||
"""
|
||||
if service_type == "prefill":
|
||||
client_idx = next(app.state.prefill_iterator)
|
||||
@@ -155,9 +153,7 @@ def get_next_client(app, service_type: str):
|
||||
async def send_request_to_service(
|
||||
client_info: dict, endpoint: str, req_data: dict, request_id: str
|
||||
):
|
||||
"""
|
||||
Send a request to a service using a client from the pool.
|
||||
"""
|
||||
"""Send a request to a service using a client from the pool."""
|
||||
req_data = req_data.copy()
|
||||
req_data["kv_transfer_params"] = {
|
||||
"do_remote_decode": True,
|
||||
@@ -200,9 +196,7 @@ async def send_request_to_service(
|
||||
async def stream_service_response(
|
||||
client_info: dict, endpoint: str, req_data: dict, request_id: str
|
||||
):
|
||||
"""
|
||||
Asynchronously stream response from a service using a client from the pool.
|
||||
"""
|
||||
"""Asynchronously stream response from a service using a client from the pool."""
|
||||
headers = {
|
||||
"Authorization": f"Bearer {os.environ.get('OPENAI_API_KEY')}",
|
||||
"X-Request-Id": request_id,
|
||||
|
||||
@@ -80,7 +80,8 @@ def _split_hnd_cache(spec) -> torch.Tensor:
|
||||
|
||||
def _packed_nhd_cache(spec) -> torch.Tensor:
|
||||
"""Logical (num_blocks, heads, block_size, 2 * head_size) over an NHD
|
||||
physical layout — the FlashAttention/FlashInfer/Triton/Flex form."""
|
||||
physical layout — the FlashAttention/FlashInfer/Triton/Flex form.
|
||||
"""
|
||||
return torch.zeros(
|
||||
NUM_BLOCKS,
|
||||
spec.block_size,
|
||||
|
||||
@@ -863,7 +863,8 @@ def test_parallelism_agnostic_excluded(kv_cache_groups: list[KVCacheGroupSpec]):
|
||||
)
|
||||
def test_canonical_layout_gate(kv_cache_groups, certified):
|
||||
"""The canonical layout certifies portability group by group; none of
|
||||
these shapes are portable in the direct layout."""
|
||||
these shapes are portable in the direct layout.
|
||||
"""
|
||||
assert not _parallelism_agnostic(kv_cache_groups)
|
||||
assert _parallelism_agnostic(kv_cache_groups, canonical=True) is certified
|
||||
|
||||
@@ -871,7 +872,8 @@ def test_canonical_layout_gate(kv_cache_groups, certified):
|
||||
def test_canonical_layout_certifies_v2_model_runner():
|
||||
"""Canonical bytes are certified per layer against live tensor strides at
|
||||
registration, so the static gate must not depend on the model-runner
|
||||
version — the v2 runner is the case the canonical layout exists for."""
|
||||
version — the v2 runner is the case the canonical layout exists for.
|
||||
"""
|
||||
groups = _groups(_full_attention_spec())
|
||||
assert _parallelism_agnostic(groups, canonical=True, v2=True)
|
||||
|
||||
|
||||
@@ -117,7 +117,8 @@ def _spec_cls_with_metric_definitions(
|
||||
metric_definitions: dict[str, Any],
|
||||
) -> type:
|
||||
"""Build a fake offloading spec class reporting the given metric
|
||||
definitions, so tests don't need to patch the real CPU spec."""
|
||||
definitions, so tests don't need to patch the real CPU spec.
|
||||
"""
|
||||
|
||||
class _FakeOffloadingSpec:
|
||||
@staticmethod
|
||||
@@ -187,7 +188,8 @@ def test_build_kv_connector_stats_with_empty_dict():
|
||||
|
||||
def test_build_kv_connector_stats_reconstructs_offload_stats():
|
||||
"""Test that OffloadingConnector stats are properly reconstructed with
|
||||
correct data."""
|
||||
correct data.
|
||||
"""
|
||||
serialized_data = {
|
||||
_StatsKey.TYPES: {
|
||||
LOAD_BYTES: _MetricType.COUNTER,
|
||||
|
||||
@@ -1583,7 +1583,8 @@ _SCAN_BEHAVIOR = {
|
||||
@pytest.mark.parametrize("result", list(LookupResult))
|
||||
def test_scan_behavior_declared_for_every_lookup_result(result: LookupResult):
|
||||
"""Whether a result keeps a sliding-window streak alive decides how wide
|
||||
the demanded chunk set gets, so every member needs deliberate behavior."""
|
||||
the demanded chunk set gets, so every member needs deliberate behavior.
|
||||
"""
|
||||
assert result in _SCAN_BEHAVIOR, f"{result} has no declared scan behavior"
|
||||
expected_lookups, expected_end = _SCAN_BEHAVIOR[result]
|
||||
|
||||
@@ -2034,7 +2035,8 @@ def test_request_level_policy_stores_all_blocks(request_runner, async_scheduling
|
||||
|
||||
def test_loads_do_not_populate_fence_index(request_runner):
|
||||
"""Loads don't populate _block_id_to_pending_jobs (protected by
|
||||
delay_free_blocks while in flight)."""
|
||||
delay_free_blocks while in flight).
|
||||
"""
|
||||
runner = request_runner(
|
||||
blocks_per_chunk=3,
|
||||
block_size=4,
|
||||
@@ -2104,7 +2106,8 @@ def test_fence_at_update_state_after_alloc(request_runner):
|
||||
def test_fence_at_build_store_jobs(request_runner):
|
||||
"""A new prefill (no load -> update_state_after_alloc returns early)
|
||||
reusing a finished request's pending-store block is flushed by
|
||||
_build_store_jobs's fence."""
|
||||
_build_store_jobs's fence.
|
||||
"""
|
||||
runner = request_runner(
|
||||
blocks_per_chunk=1,
|
||||
block_size=4,
|
||||
@@ -2154,7 +2157,8 @@ def test_fence_at_build_store_jobs(request_runner):
|
||||
@pytest.mark.parametrize("async_scheduling", [True, False])
|
||||
def test_complete_store_called_per_job(request_runner, async_scheduling: bool):
|
||||
"""complete_store fires per-job, not deferred to request finish.
|
||||
Each call carries only that store's keys."""
|
||||
Each call carries only that store's keys.
|
||||
"""
|
||||
tokens_per_block = 4
|
||||
blocks_per_chunk = 3
|
||||
tokens_per_chunk = tokens_per_block * blocks_per_chunk
|
||||
@@ -2383,7 +2387,8 @@ def test_offload_prompt_only(request_runner, async_scheduling: bool):
|
||||
@pytest.mark.parametrize("async_scheduling", [True, False])
|
||||
def test_reset_cache(request_runner, async_scheduling: bool):
|
||||
"""reset_cache flushes in-flight loads, calls manager.reset_cache(), resets
|
||||
next_stored_chunk_idx for active requests and clears job tracking."""
|
||||
next_stored_chunk_idx for active requests and clears job tracking.
|
||||
"""
|
||||
block_size = 4
|
||||
blocks_per_chunk = 3
|
||||
tokens_per_chunk = block_size * blocks_per_chunk
|
||||
@@ -2929,7 +2934,8 @@ def test_stale_sliding_window_block_after_prepare_store_failure(
|
||||
@pytest.mark.parametrize("async_scheduling", [True, False])
|
||||
def test_skip_reading_prefix_cache(request_runner, async_scheduling: bool):
|
||||
"""When skip_reading_prefix_cache=True, the offloading connector must not
|
||||
load any blocks from CPU even if a matching prefix is cached there."""
|
||||
load any blocks from CPU even if a matching prefix is cached there.
|
||||
"""
|
||||
block_size = 4
|
||||
blocks_per_chunk = 3
|
||||
tokens_per_chunk = block_size * blocks_per_chunk
|
||||
@@ -3023,7 +3029,8 @@ def test_max_load_tokens_limits_external_load(request_runner, async_scheduling:
|
||||
|
||||
class TestEagle:
|
||||
"""Tests for Eagle/MTP speculative decoding support in the offloading
|
||||
connector scheduler — both _lookup() unit tests and integration tests."""
|
||||
connector scheduler — both _lookup() unit tests and integration tests.
|
||||
"""
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# Helpers
|
||||
@@ -3565,7 +3572,8 @@ class TestEagle:
|
||||
self, request_runner, async_scheduling: bool
|
||||
):
|
||||
"""Eagle sliding-window group stores all prompt blocks but excludes
|
||||
the trailing chunk while decoding."""
|
||||
the trailing chunk while decoding.
|
||||
"""
|
||||
block_size = 4
|
||||
sliding_window = 8
|
||||
num_gpu_blocks = 100
|
||||
@@ -3617,7 +3625,8 @@ class TestEagle:
|
||||
self, request_runner, async_scheduling: bool
|
||||
):
|
||||
"""An eagle group with a single-block prompt stores it at the end of
|
||||
prefill: prompt blocks are stable, so no tail is held back."""
|
||||
prefill: prompt blocks are stable, so no tail is held back.
|
||||
"""
|
||||
block_size = 4
|
||||
blocks_per_chunk = 1
|
||||
tokens_per_chunk = block_size * blocks_per_chunk
|
||||
@@ -4254,7 +4263,8 @@ def test_request_finished_mixed_full_attn_and_sliding_window(
|
||||
|
||||
def test_chunked_local_attention_reports_its_chunk_window():
|
||||
"""Llama 4 uses chunked local attention, which used to trip the
|
||||
FullAttentionSpec assert and kill the engine at startup."""
|
||||
FullAttentionSpec assert and kill the engine at startup.
|
||||
"""
|
||||
spec = ChunkedLocalAttentionSpec(
|
||||
block_size=16,
|
||||
num_kv_heads=2,
|
||||
@@ -4493,7 +4503,8 @@ def test_retention_interval_zero_stores_only_replay_boundary(
|
||||
def _shared_kv_mtp_config():
|
||||
"""Speculative config for a shared-group MTP model: eagle-family method
|
||||
whose drafter layer merges into a target KV-cache group, so no group
|
||||
self-identifies as a drafter group (issue #52735)."""
|
||||
self-identifies as a drafter group (issue #52735).
|
||||
"""
|
||||
spec = MagicMock(name="shared_kv_mtp_spec")
|
||||
spec.use_eagle.return_value = True
|
||||
spec.use_eagle_block_drop.return_value = True
|
||||
@@ -4507,12 +4518,14 @@ def _shared_kv_mtp_config():
|
||||
class TestSharedGroupMTPOffload:
|
||||
"""Regression tests for issue #52735: OffloadingConnector must keep
|
||||
serving when speculative decoding is enabled but no KV-cache group is
|
||||
annotated as a drafter group (shared-group MTP models)."""
|
||||
annotated as a drafter group (shared-group MTP models).
|
||||
"""
|
||||
|
||||
def test_no_annotation_marks_no_groups(self, request_runner):
|
||||
"""Spec decode on + zero annotated groups must NOT mark every group
|
||||
as a drafter group; the full store->load roundtrip must match the
|
||||
non-speculative behavior of test_two_groups_full_and_sliding_window."""
|
||||
non-speculative behavior of test_two_groups_full_and_sliding_window.
|
||||
"""
|
||||
block_size = 4
|
||||
kv_cache_groups = [
|
||||
KVCacheGroupSpec(
|
||||
@@ -4569,7 +4582,8 @@ class TestSharedGroupMTPOffload:
|
||||
class TestMambaHybridOffloadServing:
|
||||
"""Store->finish->lookup flows on a full-attention + mamba-align hybrid,
|
||||
with a manager that only HITs keys that were actually stored. Guards the
|
||||
two collapse routes of issue #52735."""
|
||||
two collapse routes of issue #52735.
|
||||
"""
|
||||
|
||||
BLOCK = 4
|
||||
MAMBA_BLOCK = 16
|
||||
@@ -4698,7 +4712,8 @@ class TestMambaHybridOffloadServing:
|
||||
"""Route 1 of #52735: with spec decode on and no drafter annotation,
|
||||
serving must equal the non-speculative baseline (was 0 before the
|
||||
fix: the all-groups fallback marked the mamba group as a drafter and
|
||||
the volatile-tail pop consumed its only servable chunk)."""
|
||||
the volatile-tail pop consumed its only servable chunk).
|
||||
"""
|
||||
baseline = self._roundtrip_served_tokens(self._make_scheduler(None))
|
||||
assert baseline == 16
|
||||
served = self._roundtrip_served_tokens(
|
||||
@@ -4711,7 +4726,8 @@ class TestMambaHybridOffloadServing:
|
||||
group must not drag the confirmed boundary below a coarser sibling's
|
||||
chunk granularity. The widened query makes the volatile-tail pop
|
||||
land on an extra queried chunk, holding the boundary at 16 tokens
|
||||
(was 0 before the fix: 4 hits -> pop -> 12 tokens < mamba chunk)."""
|
||||
(was 0 before the fix: 4 hits -> pop -> 12 tokens < mamba chunk).
|
||||
"""
|
||||
scheduler = self._make_scheduler(None, mamba_eagle=True)
|
||||
assert [c.is_eagle_group for c in scheduler.config.kv_group_configs] == [
|
||||
True,
|
||||
|
||||
@@ -80,7 +80,8 @@ def _allocate_kv_caches(
|
||||
|
||||
def _single_rank_vllm_config(total_kv_heads: int):
|
||||
"""A one-rank (TP=1) parallel config, as canonical mappings are derived
|
||||
from it."""
|
||||
from it.
|
||||
"""
|
||||
vllm_config = MagicMock()
|
||||
parallel_config = vllm_config.parallel_config
|
||||
parallel_config.tensor_parallel_size = 1
|
||||
@@ -98,9 +99,7 @@ def _make_worker(
|
||||
replicated_layout: bool = False,
|
||||
rank: int = 0,
|
||||
):
|
||||
"""
|
||||
Create an OffloadingConnectorWorker with mocked dependencies.
|
||||
"""
|
||||
"""Create an OffloadingConnectorWorker with mocked dependencies."""
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.worker import (
|
||||
OffloadingConnectorWorker,
|
||||
)
|
||||
|
||||
@@ -475,8 +475,7 @@ class RequestRunner:
|
||||
complete_transfers: bool,
|
||||
post_step_fn: Callable[[], None] | None = None,
|
||||
):
|
||||
"""
|
||||
Runs multiple engine (scheduler + worker) steps.
|
||||
"""Runs multiple engine (scheduler + worker) steps.
|
||||
Assumes a single request is running.
|
||||
|
||||
Args:
|
||||
@@ -484,8 +483,8 @@ class RequestRunner:
|
||||
complete_transfers: complete transfers immediately
|
||||
post_step_fn: optional callback invoked after each step's
|
||||
update_from_output(), before the next schedule().
|
||||
"""
|
||||
|
||||
"""
|
||||
tokens_iter = iter(decoded_tokens)
|
||||
token_id = next(tokens_iter, None)
|
||||
prev_scheduler_output = None
|
||||
@@ -617,8 +616,7 @@ class RequestRunner:
|
||||
expected_flushed: tuple[int | tuple[int, int], ...] = (),
|
||||
post_step_fn: Callable[[], None] | None = None,
|
||||
):
|
||||
"""
|
||||
Runs multiple engine (scheduler + worker) steps.
|
||||
"""Runs multiple engine (scheduler + worker) steps.
|
||||
Assumes a single request is running.
|
||||
|
||||
Args:
|
||||
@@ -634,8 +632,8 @@ class RequestRunner:
|
||||
A GPU block is either a (group_idx: int, request_block_offset: int)
|
||||
or just request_block_offset: int.
|
||||
The latter case is a convenience for representing all groups.
|
||||
"""
|
||||
|
||||
"""
|
||||
expected_stored_gpu_blocks = self._to_gpu_blocks(expected_stored)
|
||||
expected_loaded_gpu_blocks = self._to_gpu_blocks(expected_loaded)
|
||||
expected_flushed_gpu_blocks = self._to_gpu_blocks(expected_flushed)
|
||||
|
||||
@@ -148,7 +148,8 @@ def test_multiturn_lifecycle():
|
||||
(do_remote_decode=True). Finishes LENGTH_CAPPED with remote_block_ids.
|
||||
Turn 2: P receives remote_block_ids from D. P pulls KV from D because
|
||||
remote_block_ids is not None and external tokens > 0. Computes only
|
||||
new tokens, finishes LENGTH_CAPPED."""
|
||||
new tokens, finishes LENGTH_CAPPED.
|
||||
"""
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG,
|
||||
)
|
||||
@@ -194,7 +195,8 @@ def test_multiturn_lifecycle():
|
||||
|
||||
def test_first_turn_no_remote_blocks():
|
||||
"""First turn: P has no remote_block_ids from D yet.
|
||||
Standard local prefill, returns kv_transfer_params for future turns."""
|
||||
Standard local prefill, returns kv_transfer_params for future turns.
|
||||
"""
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG,
|
||||
)
|
||||
@@ -274,7 +276,8 @@ def test_abort_p_side_non_length_capped():
|
||||
|
||||
def test_remote_blocks_exceed_prompt_tokens():
|
||||
"""D provides more remote tokens than P's prompt needs.
|
||||
P caps external tokens to prompt length."""
|
||||
P caps external tokens to prompt length.
|
||||
"""
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG,
|
||||
)
|
||||
@@ -310,7 +313,8 @@ def test_remote_blocks_exceed_prompt_tokens():
|
||||
def test_p_node_pulls_partial_last_block_from_d():
|
||||
"""D sends remote_block_ids with partially filled last block.
|
||||
remote_num_tokens < len(remote_block_ids) * block_size.
|
||||
P pulls only remote_num_tokens worth of external tokens."""
|
||||
P pulls only remote_num_tokens worth of external tokens.
|
||||
"""
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG,
|
||||
)
|
||||
@@ -355,7 +359,8 @@ def test_p_node_pulls_partial_last_block_from_d():
|
||||
|
||||
def test_add_new_req_to_recv_populates_remote_meta():
|
||||
"""add_new_req_to_recv correctly populates RemoteMeta for P-node
|
||||
bi-directional KV pull from D."""
|
||||
bi-directional KV pull from D.
|
||||
"""
|
||||
meta = NixlConnectorMetadata()
|
||||
kv_params = {
|
||||
"remote_block_ids": [[0, 1, 2]],
|
||||
@@ -383,7 +388,8 @@ def test_add_new_req_to_recv_populates_remote_meta():
|
||||
|
||||
def test_build_connector_meta_recv_entries():
|
||||
"""P-node scheduler: do_remote_decode=True + remote_block_ids →
|
||||
_reqs_need_recv populated, build_connector_meta produces reqs_to_recv."""
|
||||
_reqs_need_recv populated, build_connector_meta produces reqs_to_recv.
|
||||
"""
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG,
|
||||
)
|
||||
@@ -457,7 +463,8 @@ def test_p_node_pull_kv_from_d(dist_init):
|
||||
)
|
||||
def test_p_node_pull_then_send_kv(dist_init):
|
||||
"""Full P-node bi-directional: pull KV from D → prefill →
|
||||
send KV back to D via notification."""
|
||||
send KV back to D via notification.
|
||||
"""
|
||||
connector, worker = _make_connector_with_fake_worker()
|
||||
meta = _make_p_node_recv_metadata("req-p2", [10, 11], [20, 21])
|
||||
_do_load_kv(connector, meta)
|
||||
@@ -504,7 +511,8 @@ def test_p_node_deferred_pull_on_no_handshake(dist_init):
|
||||
def test_d_node_request_finished_returns_kv_params():
|
||||
"""D-node request_finished returns kv_transfer_params with
|
||||
do_remote_decode=True, remote_block_ids, remote_num_tokens
|
||||
for P to pull. These params go directly to P node."""
|
||||
for P to pull. These params go directly to P node.
|
||||
"""
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG,
|
||||
)
|
||||
@@ -585,7 +593,8 @@ def test_d_node_request_finished_remote_num_tokens():
|
||||
|
||||
def test_d_node_partial_last_block_remote_num_tokens():
|
||||
"""D-node: remote_num_tokens < len(remote_block_ids) * block_size
|
||||
when last block is partially filled."""
|
||||
when last block is partially filled.
|
||||
"""
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG,
|
||||
)
|
||||
@@ -619,7 +628,8 @@ def test_no_double_read_blocks_after_reschedule():
|
||||
bidirectional request (once on initial schedule, once after
|
||||
WAITING_FOR_REMOTE_KVS → reschedule). The _remote_blocks_processed
|
||||
flag must prevent the request from being added to _reqs_need_recv
|
||||
twice, which would cause P to read D's blocks twice."""
|
||||
twice, which would cause P to read D's blocks twice.
|
||||
"""
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG,
|
||||
)
|
||||
@@ -672,7 +682,8 @@ def test_remote_num_tokens_bounded_by_blocks():
|
||||
"""Edge case 2: D-node request_finished must return
|
||||
remote_num_tokens <= len(remote_block_ids) * block_size.
|
||||
request.num_tokens includes the last sampled token which has no KV
|
||||
in the cache, so remote_num_tokens must use num_computed_tokens."""
|
||||
in the cache, so remote_num_tokens must use num_computed_tokens.
|
||||
"""
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG,
|
||||
)
|
||||
@@ -708,7 +719,8 @@ def test_remote_num_tokens_bounded_by_blocks():
|
||||
def test_kv_recompute_threshold_skips_small_transfer():
|
||||
"""Edge case 3: When remote tokens are below kv_recompute_threshold,
|
||||
P should skip the remote pull and compute locally instead of
|
||||
entering WAITING_FOR_REMOTE_KVS."""
|
||||
entering WAITING_FOR_REMOTE_KVS.
|
||||
"""
|
||||
threshold = 256
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector_extra_config={
|
||||
@@ -750,7 +762,8 @@ def test_p_node_finished_holds_blocks_for_d():
|
||||
"""Edge case 4: P-node finishes with FINISHED_LENGTH_CAPPED and
|
||||
do_remote_decode=True. P must hold blocks (delay_free=True) and
|
||||
return kv_transfer_params with do_remote_prefill=True so D can
|
||||
read P's blocks."""
|
||||
read P's blocks.
|
||||
"""
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG,
|
||||
)
|
||||
@@ -791,7 +804,8 @@ def test_p_node_finished_holds_blocks_for_d():
|
||||
def test_cache_miss_first_turn_no_remote_pull():
|
||||
"""Edge case 5: First turn with do_remote_decode=True but no
|
||||
remote_block_ids (cache MISS). P should prefill locally with
|
||||
num_external_tokens=0 and not enter WAITING_FOR_REMOTE_KVS."""
|
||||
num_external_tokens=0 and not enter WAITING_FOR_REMOTE_KVS.
|
||||
"""
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG,
|
||||
)
|
||||
@@ -826,7 +840,8 @@ def test_cache_miss_first_turn_no_remote_pull():
|
||||
def test_partial_remote_tokens_less_than_prompt():
|
||||
"""Edge case 6: D's remote_num_tokens covers only part of P's
|
||||
prompt. P should pull remote_num_tokens worth of external tokens
|
||||
and compute the rest locally."""
|
||||
and compute the rest locally.
|
||||
"""
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG,
|
||||
)
|
||||
@@ -871,7 +886,8 @@ def test_partial_remote_tokens_less_than_prompt():
|
||||
def test_remote_blocks_processed_flag_persists():
|
||||
"""Edge case 7: After recv completes and request is rescheduled,
|
||||
the _remote_blocks_processed flag in kv_transfer_params prevents
|
||||
the bidirectional path from re-entering _reqs_need_recv."""
|
||||
the bidirectional path from re-entering _reqs_need_recv.
|
||||
"""
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG,
|
||||
)
|
||||
@@ -943,7 +959,8 @@ _REMOTE = FakeNixlConnectorWorker.REMOTE_ENGINE_ID
|
||||
)
|
||||
def test_turn2_deadline_gate(dist_init, offset, expiry_delta, expect_declined):
|
||||
"""P declines on the turn-2 readback if the deadline is known,
|
||||
near-expiry and a handshake clock offset is known"""
|
||||
near-expiry and a handshake clock offset is known
|
||||
"""
|
||||
connector, worker = _make_connector_with_fake_worker()
|
||||
worker._engine_clock_offset[_REMOTE] = offset
|
||||
expiry_time = None if expiry_delta is None else time.perf_counter() + expiry_delta
|
||||
@@ -979,7 +996,8 @@ def test_turn2_deadline_gate(dist_init, offset, expiry_delta, expect_declined):
|
||||
)
|
||||
def test_turn2_full_prefix_hit_with_expired_deadline_skips_gate(dist_init):
|
||||
"""A full local prefix hit issues no READ. The expiry gate must be skipped
|
||||
even when D's deadline is expired."""
|
||||
even when D's deadline is expired.
|
||||
"""
|
||||
connector, worker = _make_connector_with_fake_worker()
|
||||
worker._engine_clock_offset[_REMOTE] = 0.0
|
||||
|
||||
@@ -1032,7 +1050,8 @@ def test_d_node_request_finished_exports_blocks_expiry_time():
|
||||
|
||||
def test_handshake_listener_appends_perf_counter_frame():
|
||||
"""The handshake reply carries a 2nd frame with the listener's live
|
||||
perf_counter."""
|
||||
perf_counter.
|
||||
"""
|
||||
import threading
|
||||
|
||||
import msgspec
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
test that invalid blocks are evicted from prefix cache to prevent pollution.
|
||||
"""test that invalid blocks are evicted from prefix cache to prevent pollution.
|
||||
|
||||
verifies that when sync-loading fails, invalid blocks are removed from the
|
||||
prefix cache hash table so future requests cannot match and reuse corrupted data.
|
||||
@@ -39,7 +38,7 @@ def _make_get_num_new_matched_tokens(
|
||||
|
||||
@pytest.fixture
|
||||
def fail_scheduler():
|
||||
"""scheduler with kv_load_failure_policy='fail'"""
|
||||
"""Scheduler with kv_load_failure_policy='fail'."""
|
||||
vllm_config = create_vllm_config()
|
||||
vllm_config.kv_transfer_config.kv_load_failure_policy = "fail"
|
||||
return create_scheduler(vllm_config)
|
||||
@@ -48,8 +47,7 @@ def fail_scheduler():
|
||||
def test_invalid_blocks_evicted_prevents_cache_pollution(
|
||||
fail_scheduler: Scheduler,
|
||||
):
|
||||
"""
|
||||
verify invalid blocks are evicted to prevent future cache hits.
|
||||
"""Verify invalid blocks are evicted to prevent future cache hits.
|
||||
|
||||
scenario:
|
||||
1. request 1 loads externally-computed blocks (sync mode)
|
||||
|
||||
@@ -19,13 +19,15 @@ class _StubLMCacheMPConnector:
|
||||
test only asserts on the connector *name* and the ``extra_config`` dict
|
||||
produced by ``VllmConfig``, never instantiates the connector, so a bare
|
||||
placeholder class is sufficient. Not subclassing ``SupportsHMA`` mirrors
|
||||
the real connector's HMA support (it does not support HMA either)."""
|
||||
the real connector's HMA support (it does not support HMA either).
|
||||
"""
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def stub_lmcache_mp_connector(monkeypatch):
|
||||
"""Replace the lazy loader so VllmConfig.__post_init__ does not import
|
||||
``lmcache_mp_connector`` (and thus ``lmcache``) during config tests."""
|
||||
``lmcache_mp_connector`` (and thus ``lmcache``) during config tests.
|
||||
"""
|
||||
monkeypatch.setitem(
|
||||
KVConnectorFactory._registry,
|
||||
"LMCacheMPConnector",
|
||||
@@ -105,7 +107,8 @@ def _build_config(
|
||||
enable_cumem_allocator: bool = False,
|
||||
) -> VllmConfig:
|
||||
"""Build a VllmConfig that exercises _verify_kv_transfer_compat without
|
||||
requiring a real model (avoids HF downloads in CI)."""
|
||||
requiring a real model (avoids HF downloads in CI).
|
||||
"""
|
||||
from types import SimpleNamespace
|
||||
|
||||
kv_transfer_config = (
|
||||
@@ -131,7 +134,8 @@ def test_kv_connector_rejects_expandable_segments(monkeypatch, kv_connector):
|
||||
invalidated when expandable_segments lets the CUDA VMM allocator remap
|
||||
the underlying physical pages. We can't enumerate every connector that
|
||||
does this (especially OOT ones), so reject the combination whenever any
|
||||
connector is configured."""
|
||||
connector is configured.
|
||||
"""
|
||||
monkeypatch.setenv("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
|
||||
with pytest.raises(ValueError, match="expandable_segments"):
|
||||
_build_config(kv_connector=kv_connector)
|
||||
@@ -139,7 +143,8 @@ def test_kv_connector_rejects_expandable_segments(monkeypatch, kv_connector):
|
||||
|
||||
def test_kv_connector_allows_expandable_segments_with_sleep_mode(monkeypatch):
|
||||
"""Sleep mode routes KV allocations through CuMemAllocator's pool, which
|
||||
auto-disables expandable_segments (see #40812)."""
|
||||
auto-disables expandable_segments (see #40812).
|
||||
"""
|
||||
monkeypatch.setenv("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
|
||||
_build_config(kv_connector="NixlConnector", enable_sleep_mode=True)
|
||||
|
||||
@@ -162,7 +167,8 @@ def test_kv_connector_allows_other_alloc_conf(monkeypatch):
|
||||
|
||||
def test_no_kv_connector_ignores_expandable_segments(monkeypatch):
|
||||
"""The expandable_segments check only applies when a KV connector is
|
||||
configured."""
|
||||
configured.
|
||||
"""
|
||||
monkeypatch.setenv("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
|
||||
_build_config(kv_connector=None)
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user