Add pydocstyle to the ruff rules (#52136)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Signed-off-by: Harry Mellor <hej.mellor@gmail.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-09-16 16:44:29 +00:00
committed by GitHub
co-authored by Claude Opus 5
parent 804e5377aa
commit 8c1557a79c
2002 changed files with 15024 additions and 14269 deletions
@@ -151,6 +151,7 @@ def create_and_prepopulate_kv_cache(
Returns:
A 4D tensor in logical ``(num_blocks, num_kv_heads, block_size,
2 * head_size)`` order with strides determined by ``layout``.
"""
batch_size = len(k_contexts)
seq_lens = common_attn_metadata.seq_lens.cpu()
@@ -286,7 +287,6 @@ def run_attention_backend(
use_cuda_graph: bool = False,
) -> torch.Tensor:
"""Run attention computation using the specified backend's AttentionImpl."""
use_direct_block_mask = is_torch_equal_or_newer("2.9.0.dev0")
if backend == "FLEX_ATTENTION_SLOW":
use_direct_block_mask = False
@@ -414,8 +414,7 @@ def _test_backend_correctness(
max_num_seqs: int | None = None,
max_num_batched_tokens: int | None = None,
):
"""
Test that all backends produce similar outputs to a reference implementation
"""Test that all backends produce similar outputs to a reference implementation
using FlexAttention or an explicit attention-sink reference.
This test works by:
@@ -1393,7 +1392,8 @@ def test_non_causal_backend_correctness(
model: str,
):
"""Test backend's correctness with non-causal (bidirectional) decoder
attention, as used by DFlash speculative decoding."""
attention, as used by DFlash speculative decoding.
"""
def bidirectional_mask_mod(
b: torch.Tensor,
@@ -115,7 +115,8 @@ def test_mamba_layers_have_unified_interface(
layer_class, expected_backend, expected_mamba_type
):
"""Test that all Mamba layers have the unified get_attn_backend
interface."""
interface.
"""
assert hasattr(layer_class, "get_attn_backend"), (
f"{layer_class.__name__} should have get_attn_backend method"
)
+16 -15
View File
@@ -20,12 +20,12 @@ from vllm.v1.worker.ubatch_utils import (
@pytest.fixture
def sample_query_start_loc():
"""Sample query_start_loc tensor for testing"""
"""Sample query_start_loc tensor for testing."""
return torch.tensor([0, 5, 12, 20, 35, 50])
def test_basic_slice_middle(sample_query_start_loc):
"""Test slicing from middle of tensor"""
"""Test slicing from middle of tensor."""
req_slice = slice(1, 3) # slice from index 1 to 3
result = slice_query_start_locs(sample_query_start_loc, req_slice)
@@ -34,7 +34,7 @@ def test_basic_slice_middle(sample_query_start_loc):
def test_slice_from_beginning(sample_query_start_loc):
"""Test slicing from the beginning of tensor"""
"""Test slicing from the beginning of tensor."""
req_slice = slice(0, 2) # slice from index 0 to 2
result = slice_query_start_locs(sample_query_start_loc, req_slice)
@@ -43,7 +43,7 @@ def test_slice_from_beginning(sample_query_start_loc):
def test_slice_to_end(sample_query_start_loc):
"""Test slicing to the end of tensor"""
"""Test slicing to the end of tensor."""
req_slice = slice(3, 5) # slice from index 3 to 5 (last index)
result = slice_query_start_locs(sample_query_start_loc, req_slice)
@@ -52,7 +52,7 @@ def test_slice_to_end(sample_query_start_loc):
def test_single_element_slice(sample_query_start_loc):
"""Test slice that results in single element"""
"""Test slice that results in single element."""
req_slice = slice(2, 3) # slice from index 2 to 3
result = slice_query_start_locs(sample_query_start_loc, req_slice)
@@ -61,7 +61,7 @@ def test_single_element_slice(sample_query_start_loc):
def test_full_tensor_slice(sample_query_start_loc):
"""Test slicing the entire tensor"""
"""Test slicing the entire tensor."""
req_slice = slice(0, 5) # slice entire tensor
result = slice_query_start_locs(sample_query_start_loc, req_slice)
@@ -80,7 +80,7 @@ def test_slice_bounds_edge_cases(sample_query_start_loc):
@pytest.fixture
def small_decode_metadata():
"""Create metadata for small decode batch"""
"""Create metadata for small decode batch."""
batch_spec = BATCH_SPECS["small_decode"]
device = torch.device("cpu")
return create_common_attn_metadata(batch_spec, block_size=16, device=device)
@@ -88,7 +88,7 @@ def small_decode_metadata():
@pytest.fixture
def large_decode_metadata():
"""Create metadata for small decode batch"""
"""Create metadata for small decode batch."""
batch_spec = BATCH_SPECS["large_decode"]
device = torch.device("cpu")
return create_common_attn_metadata(batch_spec, block_size=16, device=device)
@@ -96,7 +96,7 @@ def large_decode_metadata():
@pytest.fixture
def mixed_small_metadata():
"""Create metadata for mixed small batch"""
"""Create metadata for mixed small batch."""
batch_spec = BATCH_SPECS["mixed_small"]
device = torch.device("cpu")
return create_common_attn_metadata(batch_spec, block_size=16, device=device)
@@ -104,7 +104,7 @@ def mixed_small_metadata():
# Tests for _make_metadata_with_slice
def test_make_metadata_with_slice_decode_batch(small_decode_metadata):
"""Test slicing decode batch metadata"""
"""Test slicing decode batch metadata."""
# Split first request only
ubatch_slice = UBatchSlice(slice(0, 1), slice(0, 1))
@@ -119,7 +119,7 @@ def test_make_metadata_with_slice_decode_batch(small_decode_metadata):
def test_make_metadata_with_slice_mixed_batch(mixed_small_metadata):
"""Test slicing mixed batch metadata"""
"""Test slicing mixed batch metadata."""
ubatch_slice = UBatchSlice(slice(1, 3), slice(1, 7)) # Requests 1-3, tokens 1-7
result = _make_metadata_with_slice(ubatch_slice, mixed_small_metadata)
@@ -132,7 +132,7 @@ def test_make_metadata_with_slice_mixed_batch(mixed_small_metadata):
def test_split_attn_metadata_decode_batch(large_decode_metadata):
"""Test splitting decode batch into two equal parts"""
"""Test splitting decode batch into two equal parts."""
num_tokens = large_decode_metadata.num_reqs
mid_point = num_tokens // 2
ubatch_slices = [
@@ -164,7 +164,8 @@ def apply_split_decodes_and_prefills(
treat_short_extends_as_decodes: bool = True,
):
"""Helper function to apply split_decodes_and_prefills and return
the results."""
the results.
"""
device = torch.device("cpu")
seq_lens = [10 * (i + 1) for i in range(len(query_lens))]
common_metadata = create_common_attn_metadata(
@@ -297,7 +298,7 @@ def test_split_decodes_and_prefills_uniform_mixed_batch_non_uniform_decodes():
def test_split_decodes_and_prefills_uniform_padded_batch_all_same():
"""uniform batch where all query lengths are identical with 0 length padded reqs."""
"""Uniform batch where all query lengths are identical with 0 length padded reqs."""
# All query lengths are 2, with decode_threshold=3 (so 2 <= 3)
# This triggers the padded uniform path at line 891
query_lens = [2, 2, 2, 0]
@@ -324,7 +325,7 @@ def test_split_decodes_and_prefills_uniform_padded_batch_all_same():
def test_prefill_split_across_ubatches(
seq_lens, query_lens, split_point, expected_first_reqs, expected_second_reqs
):
"""Test splitting a prefill across ubatches"""
"""Test splitting a prefill across ubatches."""
import numpy as np
device = torch.device("cpu")
+2 -1
View File
@@ -1,7 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""The empty-shard mask fused into the DCP A2A pack kernel must be bit-exact
with the eager ``mask_dcp_empty_shards_`` pass it replaces."""
with the eager ``mask_dcp_empty_shards_`` pass it replaces.
"""
import pytest
import torch
@@ -495,7 +495,8 @@ def test_dspark_noncausal_sparse_mla_matches_sdpa(
num_heads,
):
"""Non-causal (window ∪ block, future-pointing) per-token indices must match
a dense SDPA reference over the same indices, for both sparse-MLA backends."""
a dense SDPA reference over the same indices, for both sparse-MLA backends.
"""
_skip_if_backend_unavailable(backend_cls, kv_cache_dtype, block_size)
window, block, topk_width = _DSPARK_CONFIGS[config_name]
@@ -231,7 +231,8 @@ def test_plan_uses_state_params(monkeypatch):
def test_kv_lens_host_formula():
"""Per-row host lengths: context == position + 1; capped at
index_topk + trailing-pool remainder past the sparse threshold."""
index_topk + trailing-pool remainder past the sparse threshold.
"""
builder = object.__new__(FlashInferMLASparseSM90Builder)
builder._index_topk = 2048
builder._index_kpool = 4
@@ -187,7 +187,8 @@ def test_gdn_build_classification(test_case: GDNBuildTestCase):
def test_has_initial_state_after_reclassification():
"""After reclassification, num_prefills > 0 so the prefill kernel path
should compute has_initial_state. For the reclassified request with
context_lens > 0, the corresponding entry must be True."""
context_lens > 0, the corresponding entry must be True.
"""
builder = _create_gdn_builder(num_speculative_tokens=2)
batch = BatchSpec(seq_lens=[65, 20], query_lens=[1, 3])
meta = _build(builder, batch, num_decode_draft_tokens=[-1, 2])
@@ -56,7 +56,8 @@ def _ref_stable_topk_from_candidates_fp64(
) -> torch.Tensor:
"""Pure-PyTorch reference for the CuteDSL stable-topk selector order
(score desc, then lowest global token id). Selects the same SET as the
kernel; only the set is compared in tests."""
kernel; only the set is compared in tests.
"""
num_rows, num_candidates = candidate_scores.shape
device = candidate_scores.device
select_k = min(k, num_candidates)
@@ -121,7 +122,8 @@ def _dcp_lse_merge(
class _FakeDCPGroup:
"""Single-process stand-in: ``all_gather`` returns the pre-built
concatenation of every rank's packed ``(score, global_id)`` candidates,
mirroring the one packed all-gather the merge issues."""
mirroring the one packed all-gather the merge issues.
"""
def __init__(self, gathered_packed: torch.Tensor) -> None:
self.gathered_packed = gathered_packed
@@ -207,7 +209,8 @@ def _merge_local_topks_global_with_fake_dcp(
"""Run ``_merge_dcp_topk_global`` for every rank against a faked
all-gather and return each rank's global-index result (all ranks should
agree). The fake pre-builds the packed candidate concatenation the merge's
single ``all_gather`` would return."""
single ``all_gather`` would return.
"""
# The merge is now CuteDSL-only (no PyTorch fallback), so it runs the real
# Triton pack + CuteDSL selector kernels even behind the faked all-gather.
if not current_platform.is_cuda() or not has_cutedsl():
@@ -709,7 +712,8 @@ def test_dcp_filter_compaction_matches_reference(
"""In-kernel compaction must, for every row, produce exactly the rank-owned
physical slots packed into [0, valid_count) with -1 in the tail -- the same
SET a reference filter + sort/gather produces. Uses wide rows (> BLOCK_N
valid slots), with interior -1 gaps."""
valid slots), with interior -1 gaps.
"""
device = torch.device("cuda")
torch.manual_seed(7)
dcp_size = 2
@@ -867,7 +871,8 @@ def test_decode_topk_pads_surplus_with_negative_one():
so a non-(-1) pad would be silently attended. This is the common case under
DCP, where each rank's local seq_len = global / world is usually << topk.
Checked on the *set* of valid indices (the kernel may order them by score,
not position)."""
not position).
"""
torch.manual_seed(0)
device = torch.device("cuda")
topk = 8
@@ -913,7 +918,8 @@ def test_sparse_decode_dcp_short_context_matches_non_dcp():
"""End-to-end DCP decode where the global seq_len < topk (so every rank's
local top-k is surplus-padded). Exercises the kernel surplus -> merge mask
-> global top-k -> physical localize -> LSE merge chain for the common
short-context decode case, vs the non-DCP reference."""
short-context decode case, vs the non-DCP reference.
"""
torch.manual_seed(4)
device = torch.device("cuda")
world = 2
@@ -58,7 +58,8 @@ def test_tail_backend_layout_matches_kernel_pointer_arithmetic():
def make_tail_block_table(own_blocks, width=64):
"""Tail-group block table as BlockTables produces it: column 0 holds the
request's single KpoolTailManager block, the remaining columns are never
written and stay zero."""
written and stay zero.
"""
bt = torch.zeros(len(own_blocks), width, dtype=torch.int32)
bt[:, 0] = torch.tensor(own_blocks, dtype=torch.int32)
return bt
@@ -66,7 +67,8 @@ def make_tail_block_table(own_blocks, width=64):
def legacy_generic_tail_slots(block_table, query_start_loc, positions):
"""Reference of the generic ``_compute_slot_mappings_kernel`` arithmetic
(block_table.py:305-313) applied to the tail group's table."""
(block_table.py:305-313) applied to the tail group's table.
"""
slots = []
for req in range(block_table.shape[0]):
for i in range(query_start_loc[req], query_start_loc[req + 1]):
@@ -107,7 +109,8 @@ def make_batch(per_req_positions, padded_len=None):
def test_legacy_generic_mapping_collapses_onto_block_zero():
"""The bug: with the manager's 1-column block table, the generic kernel
maps every pos >= kpool onto tail block 0, and distinct requests collide."""
maps every pos >= kpool onto tail block 0, and distinct requests collide.
"""
own_blocks = [5, 9]
per_req = [list(range(10)), list(range(12))] # prompts of len 10 and 12
positions, qsl, _, num_actual, num_reqs = make_batch(per_req)
@@ -137,7 +140,8 @@ def test_legacy_generic_mapping_collapses_onto_block_zero():
def test_circular_mapping_isolates_requests():
"""The fix: every token lands in its own request's block at pos % kpool,
and no slot is ever shared by two different requests (slots do recur
within a request every kpool positions -- that is the circular design)."""
within a request every kpool positions -- that is the circular design).
"""
own_blocks = [5, 9]
per_req = [list(range(10)), list(range(12))]
positions, qsl, slot_mapping, num_actual, num_reqs = make_batch(per_req)
@@ -165,7 +169,8 @@ def test_circular_mapping_isolates_requests():
def test_circular_mapping_matches_generic_for_short_requests(prompt_len):
"""For pos < kpool the generic kernel already picks the own block, so the
two mappings agree while every position fits the request's first block
(single-request behavior is unchanged)."""
(single-request behavior is unchanged).
"""
own_blocks = [7]
per_req = [list(range(prompt_len))]
positions, qsl, slot_mapping, num_actual, num_reqs = make_batch(per_req)
@@ -275,7 +280,8 @@ class TailRingMirror:
"""Mirror of _kpool_tail_seed_kernel / _kpool_decode_update_batched_kernel
addressing: block = tail_slot // kpool, ring offset = pos % kpool; a pool
completing at pos reads ring slots (pool_start + s) % kpool and uses the
current token's own K/score for the last member."""
current token's own K/score for the last member.
"""
def __init__(self, num_blocks, kpool=KPOOL):
self.kpool = kpool
@@ -318,7 +324,8 @@ def tail_slot_for(mapping, req, pos, own_block):
def run_scenario(mapping, interleave):
"""Requests A (block 5, prompt len 9) and B (block 9, prompt len 11)
decode concurrently; returns A's boundary pool [8, 9, 10, 11]."""
decode concurrently; returns A's boundary pool [8, 9, 10, 11].
"""
ring = TailRingMirror(num_blocks=16)
blocks = {"A": 5, "B": 9}
prompts = {"A": 9, "B": 11}
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Regression test for https://github.com/vllm-project/vllm/issues/34865
"""Regression test for https://github.com/vllm-project/vllm/issues/34865.
When multiple KV cache groups share the same MambaSpec (as in Nemotron
hybrid models), the metadata caching optimization reuses metadata from
@@ -29,7 +28,8 @@ def _make_vllm_config(
block_size: int | None = None,
):
"""Create a minimal mock VllmConfig with only the fields the builder
accesses, avoiding any model download / HF config inspection."""
accesses, avoiding any model download / HF config inspection.
"""
speculative_config = (
SimpleNamespace(
num_speculative_tokens=num_speculative_tokens,
@@ -75,8 +75,8 @@ def test_mamba_single_token_prompt_runs_as_prefill():
def test_update_block_table_copies_block_idx_to_persistent_buffers():
"""update_block_table() must write block_idx tensors to the current
builder's persistent buffers, not leave them pointing to a different
builder's buffers."""
builder's buffers.
"""
block_size = 16
max_model_len = 256
num_reqs = 4
@@ -180,8 +180,8 @@ def test_state_indices_tensor_d_includes_num_speculative_blocks():
bug 1: with mamba_cache_mode='all' and speculative decoding enabled,
the cudagraph buffer for state_indices_tensor_d must allocate the same
per-request column count as the runtime block table, which includes
num_speculative_blocks trailing scratch columns."""
num_speculative_blocks trailing scratch columns.
"""
block_size = 16
max_model_len = 256
max_num_seqs = 4
@@ -214,8 +214,8 @@ def test_block_idx_cudagraph_capture_padded_by_num_reqs():
bug 2: with mamba_cache_mode='all' and spec decode, _update_metadata_for
_cudagraph_capture must slice block_idx_last_{scheduled,computed}_token
by the request count (padded_bs == num_reqs), not by num_decode_tokens.
Past num_decodes, the slice must be zero-filled."""
Past num_decodes, the slice must be zero-filled.
"""
block_size = 16
max_model_len = 256
max_num_seqs = 8
@@ -295,7 +295,8 @@ def test_block_idx_prev_step_persistent_buffer_allocated():
block_idx_last_scheduled_token_prev_step as a persistent buffer with the
same shape as the existing block_idx_last_{scheduled,computed}_token
buffers, so cudagraph capture records a stable pointer for the prev-step
input anchor consumed by mamba_mixer2's input gather."""
input anchor consumed by mamba_mixer2's input gather.
"""
block_size = 16
max_model_len = 256
max_num_seqs = 8
@@ -323,7 +324,8 @@ def test_block_idx_prev_step_persistent_buffer_allocated():
def test_block_idx_prev_step_persistent_buffer_skipped_without_spec_decode():
"""Without spec decode, the prev-step buffer is unused and must not be
allocated — the input anchor reduces to last_computed_token."""
allocated — the input anchor reduces to last_computed_token.
"""
block_size = 16
max_model_len = 256
max_num_seqs = 8
@@ -347,7 +349,8 @@ def test_block_idx_prev_step_cudagraph_capture_uses_persistent_buffer():
"""_update_metadata_for_cudagraph_capture must copy the prev-step anchor
into the builder's persistent buffer (so cudagraph replay reads from the
same underlying memory), pad past num_decodes with zero, and return a
slice of the persistent buffer in the metadata."""
slice of the persistent buffer in the metadata.
"""
block_size = 16
max_model_len = 256
max_num_seqs = 8
+4 -5
View File
@@ -171,7 +171,8 @@ def test_glm5_flashinfer_masked_mha_routing(
def test_concat_k_nope_k_pe_matches_torch_cat(qk_rope_head_dim):
"""The K concat used by the MLA prefill context loop must equal torch.cat of
k_nope with the broadcast k_pe; with no RoPE part it returns k_nope itself
instead of allocating and copying."""
instead of allocating and copying.
"""
torch.manual_seed(0)
num_tokens, num_heads, qk_nope_head_dim = 5, 4, 256
k_nope = torch.randn(num_tokens, num_heads, qk_nope_head_dim, dtype=torch.bfloat16)
@@ -515,6 +516,7 @@ def create_and_prepopulate_kv_cache(
Returns:
MLA KV cache tensor
"""
batch_size = len(kv_c_contexts)
seq_lens = common_attn_metadata.seq_lens.cpu()
@@ -1307,7 +1309,6 @@ def run_attention_backend(
chunked_prefill_workspace_size: int | None = None,
) -> torch.Tensor:
"""Run attention computation using the specified backend's AttentionImpl."""
builder_cls, impl_cls = try_get_attention_backend(backend)
# Force the prefill backend selection (None means auto-select).
@@ -1434,8 +1435,7 @@ def _run_backend_correctness(
v_head_dim: int,
chunked_prefill_workspace_size: int | None = None,
):
"""
Test that all backends produce similar outputs to a reference implementation
"""Test that all backends produce similar outputs to a reference implementation
using torch.nn.functional.scaled_dot_product_attention.
This test works by:
@@ -1454,7 +1454,6 @@ def _run_backend_correctness(
multiple GPUs. This tests that backends work correctly with different
head counts.
"""
# Filter backends to those that support the requested kv_cache_dtype
backends_to_test = [
b
@@ -134,7 +134,8 @@ def test_flash_attn_registers_warmup_only_for_fa4(
def test_flash_attn_prefill_backend_signature_accepts_fused_kwargs():
"""run_prefill_new_tokens must accept out/output_scale so the direct
(non-**kwargs) call in forward_mha type- and runtime-checks."""
(non-**kwargs) call in forward_mha type- and runtime-checks.
"""
import inspect
params = inspect.signature(
@@ -150,7 +151,8 @@ def test_flash_attn_prefill_backend_signature_accepts_fused_kwargs():
def test_mla_impl_forward_mha_accepts_output_scale():
"""The abstract MLA impl forward_mha must carry output_scale so every
override (and the unconditional forward_impl call) stays compatible."""
override (and the unconditional forward_impl call) stays compatible.
"""
import inspect
from vllm.v1.attention.backend import MLAAttentionImpl
@@ -177,7 +179,8 @@ _FUSED_FP8_SKIP = _fused_fp8_skip_reason()
def test_fa4_fused_fp8_output_matches_post_quant(default_vllm_config):
"""FA4's fused FP8 write (output_scale, flash-attention#135) must match the
bf16-attention + standalone static-FP8-quant path it replaces, since
production uses the same output_scale for both."""
production uses the same output_scale for both.
"""
from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
from vllm.platforms import current_platform
@@ -315,7 +315,8 @@ class TestBackendValidation:
def test_flash_attn_accepts_glm53_flash_nope_dimensions(self):
"""(256, 0, 256) runs the same FA kernels as GLM-5's (192, 64, 256); a
RoPE-carrying 320-wide query does not."""
RoPE-carrying 320-wide query does not.
"""
try:
from vllm.v1.attention.backends.mla.prefill.flash_attn import (
FlashAttnPrefillBackend,
@@ -257,7 +257,8 @@ def test_replayssm_write_pos(case: ReplaySSMBuildCase):
def test_resumed_request_differs_from_fresh():
"""Same token count, different decode_base: fresh (base 100) -> write_pos 5,
resumed (base 105) -> write_pos 0."""
resumed (base 105) -> write_pos 0.
"""
builder = _create_replayssm_builder(16)
batch = BatchSpec(seq_lens=[106, 106], query_lens=[1, 1])
common = create_common_attn_metadata(batch, BLOCK_SIZE, DEVICE).replace(
@@ -514,7 +514,6 @@ def test_mtp_builder_init_sizes_native_fp8_metadata(
Sweeping num_heads asserts metadata is sized for the padded decode shape,
covering Kimi-K3 TP4's 24 -> 32 head path and native fp8 nhead=32 folding.
"""
dtypes = SimpleNamespace(fp8="fp8", fp16="fp16", bf16="bf16")
info_calls = []
@@ -694,7 +693,6 @@ def test_full_cudagraph_padded_uniform_mtp_synthesizes_decode_indptr(
monkeypatch,
):
"""Full-CG zero-qo rows follow rocm_aiter_mla.py:608-657,717-759."""
get_mla_metadata_v1 = mock.MagicMock()
monkeypatch.setitem(
sys.modules,
@@ -756,7 +754,6 @@ def test_full_cudagraph_padded_uniform_mtp_synthesizes_decode_indptr(
def test_decode_expands_kernel_block_page_indices(monkeypatch):
"""kernel_block_size>1 expands b -> b*K+offset at rocm_aiter_mla.py:696-704."""
expand_kernel = _ExpandPageIndicesKernel()
monkeypatch.setattr(rocm_aiter_mla, "_expand_page_indices_kernel", expand_kernel)
# qlen==1 now takes the persistent-metadata path, which imports
@@ -517,7 +517,8 @@ def test_auto_selection_for_kv_connector(
aiter_found, expected_backend, mock_vllm_config, mock_get_cdna_version
):
"""Auto-selection with a KV connector and AITER enabled resolves to unified attn,
and to triton attn if AITER not enabled."""
and to triton attn if AITER not enabled.
"""
from vllm.platforms.rocm import RocmPlatform
with patch(
@@ -102,7 +102,8 @@ def expected_row_seq_lens(per_req_positions, batch_size, next_n):
def test_legacy_flat_layout_misaligns_and_bleeds():
"""The bug: ``positions[:n] + 1`` with padded-row coordinates reads other
requests' positions and prefill positions."""
requests' positions and prefill positions.
"""
decode_lens, per_req, positions = make_non_uniform_batch()
batch_size = decode_lens.shape[0]
next_n = int(decode_lens.max())
@@ -137,7 +138,8 @@ def test_helper_uniform_layout_matches_flat_slice():
def test_helper_padded_layout_per_row():
"""Non-uniform batches map every padded row to its own token's position;
pad rows get 0 (empty tail)."""
pad rows get 0 (empty tail).
"""
decode_lens, per_req, positions = make_non_uniform_batch()
out = _decode_topk_seq_lens(positions, decode_lens, 8, 3, 4, requires_padding=True)
expected = expected_row_seq_lens(per_req, 3, 4)
@@ -149,7 +151,8 @@ def test_helper_padded_layout_per_row():
def expand_tail_region(dec_seq):
"""Run the production pure-torch expand + append pair (the fused kernel
replicates it exactly on the identity path) and return the tail columns
[TOPK_TOKENS, TOPK_TOKENS + KPOOL - 1)."""
[TOPK_TOKENS, TOPK_TOKENS + KPOOL - 1).
"""
rows = dec_seq.shape[0]
pool_ids = torch.arange(SELECT_K, dtype=torch.int64).expand(rows, SELECT_K)
valid = torch.ones_like(pool_ids, dtype=torch.bool)
@@ -163,7 +166,8 @@ def expand_tail_region(dec_seq):
def test_tail_expansion_legacy_vs_fixed():
"""End-to-end tail consequence: the fixed mapping appends exactly the
request's trailing incomplete pool; the legacy flat slice drops real tail
tokens and emits indices far past the request's own sequence."""
tokens and emits indices far past the request's own sequence.
"""
decode_lens, per_req, positions = make_non_uniform_batch()
n = 12
fixed = _decode_topk_seq_lens(
+20 -11
View File
@@ -245,8 +245,8 @@ def _dequantize_fp8_ds_mla_entry(
Args:
simulate_sm100_e8m0_scales: If True, simulate the SM100 kernel's
float -> e8m0 -> bf16 scale conversion path.
"""
"""
# The first kv_lora_rank bytes store FP8 latent values with one scale per
# 128 element tile written as float32 right after the latent payload.
scales = cache_slice.view(torch.float32)[kv_lora_rank // 4 : kv_lora_rank // 4 + 4]
@@ -283,8 +283,8 @@ def _quantize_dequantize_fp8_ds_mla(
Args:
simulate_sm100_e8m0_scales: If True, simulate the SM100 kernel's
float -> e8m0 -> bf16 scale conversion in dequantization.
"""
"""
if kv_c.numel() == 0:
return kv_c.clone(), k_pe.clone()
@@ -909,7 +909,8 @@ def test_index_group_convert_during_piecewise_capture():
runs in the following eager break. The convert's side stream must not
wait on ``logical_topk_ready`` there: an event recorded inside a captured
segment is graph-local, and an eager wait on it raises
cudaErrorInvalidValue."""
cudaErrorInvalidValue.
"""
device = torch.device(DEVICE_TYPE)
num_tokens, num_topk, num_requests, blocks_per_req, block_size = 8, 128, 4, 4, 16
@@ -1048,7 +1049,8 @@ def test_triton_convert_rejects_req_id_longer_than_token_indices():
req_id but the output is allocated like token_indices, so a full-batch
req_id combined with an MQA-subset token_indices wrote past the end of
the output buffer. The wrapper must reject the length mismatch instead
of corrupting memory."""
of corrupting memory.
"""
device = torch.device(DEVICE_TYPE)
num_topk_tokens = 128
block_size = 64
@@ -1103,7 +1105,8 @@ def test_flashmla_forward_bf16_kv_slices_req_id_to_mqa_tokens():
is active, forward_mqa only receives the leading decode tokens, but
_forward_bf16_kv passed the full-batch req_id_per_token to the index
conversion, making it write past the end of its output buffer. The call
site must slice req_id_per_token to the MQA tokens."""
site must slice req_id_per_token to the MQA tokens.
"""
device = torch.device(DEVICE_TYPE)
num_topk_tokens = 128
block_size = 64
@@ -1201,7 +1204,8 @@ def test_split_prefill_chunks(seq_lens, max_buf, expected):
)
def test_masked_mha_workspace_fits_single_request_boundary(max_query_len, expected):
"""A 32K prefill needs the default workspace exactly; shrinking it would
push a supported request onto MQA."""
push a supported request onto MQA.
"""
assert (
_masked_mha_workspace_fits(
batch_size=1,
@@ -1259,7 +1263,8 @@ def test_is_masked_mha_available_model_dims(
):
"""The allow-list gates masked MHA per exact model geometry: the DeepSeek-V3.2,
GLM-5 and NoPE GLM-5.3-Flash layouts on an SM100-family GPU with FA4 and an
unquantized KV cache, nothing else."""
unquantized KV cache, nothing else.
"""
import vllm.model_executor.layers.attention.sparse_mla_attention as mod
monkeypatch.setattr(
@@ -3599,7 +3604,8 @@ def test_fp8_mixed_batch_dcp_neutralizes_empty_rows(monkeypatch):
"""A decode row whose top-k shard holds no local candidates (all -1) has
undefined kernel out/lse; it must come back as (0, -inf), the identity of
the cross-rank LSE merge, or a NaN would survive the merge even at zero
weight (0 * NaN = NaN)."""
weight (0 * NaN = NaN).
"""
num_tokens, num_heads, head_dim = 3, 2, 3
q = torch.empty(num_tokens, num_heads, head_dim, device=DEVICE_TYPE)
local_indices = torch.tensor(
@@ -3772,7 +3778,8 @@ def test_hisparse_fp8_prefill_gather_uses_dedicated_stream(monkeypatch):
def test_sparse_impl_observes_repointed_indexer_buffer():
"""The MTP proposer repoints the draft's indexer at the target model's buffer
after the backend impl is built, so the impl must resolve the buffer per read.
Snapshotting it in __init__ leaves the layer reading indices nothing writes."""
Snapshotting it in __init__ leaves the layer reading indices nothing writes.
"""
impl = object.__new__(FlashInferMLASparseImpl)
own = torch.zeros(4, 8, dtype=torch.int32)
target = torch.ones(4, 8, dtype=torch.int32)
@@ -3788,7 +3795,8 @@ def test_sparse_impl_observes_repointed_indexer_buffer():
def test_explicit_topk_buffer_supersedes_indexer():
"""Backbone skip-topk layers have no indexer, and the proposer also assigns the
shared buffer directly onto draft submodules."""
shared buffer directly onto draft submodules.
"""
impl = object.__new__(FlashInferMLASparseImpl)
indexer = SimpleNamespace(
topk_indices_buffer=torch.zeros(2, 2, dtype=torch.int32),
@@ -3803,7 +3811,8 @@ def test_explicit_topk_buffer_supersedes_indexer():
def test_sparse_mla_common_impl_resolves_buffer_lazily():
"""Guards the whole SparseMLACommonImpl family at once: a backend that
snapshots the buffer instead silently loses MTP buffer sharing."""
snapshots the buffer instead silently loses MTP buffer sharing.
"""
assert issubclass(SparseMLACommonImpl, SharedTopkIndicesBuffer)
assert isinstance(SparseMLACommonImpl.topk_indices_buffer, property), (
"topk_indices_buffer must stay a lazily-resolved property"
@@ -124,6 +124,7 @@ def _create_nvfp4_hnd_kv_cache(
Returns:
``torch.Tensor``: The nvfp4 kv_cache tensor (uint8, LBHNC-strided).
"""
# First create a bf16 cache so block tables are populated.
bf16_cache = create_and_prepopulate_kv_cache(
@@ -197,7 +198,8 @@ def _create_nvfp4_hnd_kv_cache(
def _run_trtllm_integration(batch_spec, kv_cache_dtype="auto", model_name=MODEL):
"""Run TRTLLM attention through the full FlashInfer pipeline
and compare against an SDPA reference."""
and compare against an SDPA reference.
"""
set_random_seed(42)
device = torch.device(f"{DEVICE_TYPE}:0")
@@ -425,7 +427,8 @@ def _run_trtllm_integration(batch_spec, kv_cache_dtype="auto", model_name=MODEL)
def test_trtllm_gen_full_attention_integration(batch_spec_name: str):
"""Test TRTLLM gen-full attention through the full FlashInfer
MetadataBuilder.build() -> FlashInferImpl.forward() pipeline,
with real TRTLLM kernels on Blackwell."""
with real TRTLLM kernels on Blackwell.
"""
_run_trtllm_integration(BATCH_SPECS[batch_spec_name])
@@ -436,7 +439,8 @@ def test_trtllm_gen_full_attention_integration(batch_spec_name: str):
@torch.inference_mode()
def test_trtllm_gen_nvfp4_kv_integration(batch_spec_name: str):
"""Test TRTLLM attention with nvfp4 KV cache through the full
FlashInfer MetadataBuilder.build() -> FlashInferImpl.forward() pipeline."""
FlashInfer MetadataBuilder.build() -> FlashInferImpl.forward() pipeline.
"""
_run_trtllm_integration(
BATCH_SPECS[batch_spec_name],
kv_cache_dtype="nvfp4",
-1
View File
@@ -193,7 +193,6 @@ def create_vllm_config(
hf_config_override: dict | None = None,
) -> VllmConfig:
"""Create a VllmConfig for testing with reasonable defaults."""
model_config = ModelConfig(
model=model_name,
tokenizer=model_name,