vLLM/tests/v1/attention/test_sparse_mla_backends.py

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for the sparse MLA backends and utilities."""

import math
from collections import deque
from types import MethodType, SimpleNamespace
from unittest.mock import MagicMock

import pytest
import torch

from tests.v1.attention.test_mla_backends import (
    BATCH_SPECS,
    BatchSpec,
    MockSparseMLAAttentionLayer,
    create_and_prepopulate_kv_cache,
)
from tests.v1.attention.utils import (
    create_common_attn_metadata,
    create_standard_kv_cache_spec,
    create_vllm_config,
)
from vllm import _custom_ops as ops
from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphCapture
from vllm.config import (
    CUDAGraphMode,
    HiSparseConfig,
    SpeculativeConfig,
    set_current_vllm_config,
)
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.worker import (
    HiSparseConnectorWorker,
)
from vllm.forward_context import set_forward_context
from vllm.model_executor.layers.attention import mla_attention
from vllm.model_executor.layers.attention.mla_attention import _use_masked_mha
from vllm.model_executor.layers.attention.sparse_mla_attention import (
    GLOBAL_TOPK_MASK_MAX_BYTES,
    SparseMLACommonImpl,
    SparseMLAPrefillMetadata,
    _is_masked_mha_available,
    _masked_mha_workspace_fits,
    _topk_mask_shape,
    _use_dense_mha_prefill,
)
from vllm.model_executor.layers.linear import ColumnParallelLinear
from vllm.platforms import current_platform

# TODO: Integrate ROCMAiterMLASparseBackend for ROCm.
# The ROCm sparse MLA backend (rocm_aiter_mla_sparse.py) has a compatible
# forward_mqa interface but needs validation on ROCm hardware.
if not current_platform.is_cuda():
    pytest.skip(
        "Sparse MLA backend tests currently only support CUDA. "
        "ROCm support requires integrating ROCMAiterMLASparseBackend.",
        allow_module_level=True,
    )

from vllm.model_executor.layers.attention.mla_attention import (
    _canonicalize_sparse_mla_kv_cache_dtype,
)
from vllm.model_executor.layers.attention.sparse_mla_attention import (
    SharedTopkIndicesBuffer,
)
from vllm.utils.math_utils import cdiv
from vllm.utils.torch_utils import current_stream
from vllm.v1.attention.backends.mla import index_group as index_group_module
from vllm.v1.attention.backends.mla.flashattn_mla_sparse import (
    FlashAttnMLASparseImpl,
)
from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import (
    FlashInferMLASparseImpl,
    FlashInferMLASparseMetadataBuilder,
    FlashInferMLASparseTRTLLMBackend,
)
from vllm.v1.attention.backends.mla.flashinfer_mla_sparse_sm120 import (
    FlashInferMLASparseSM120Impl,
)
from vllm.v1.attention.backends.mla.flashmla_sparse import (
    FlashMLASparseBackend,
    FlashMLASparseImpl,
    FlashMLASparseMetadata,
    FlashMLASparseMetadataBuilder,
    triton_convert_req_index_to_global_index,
)
from vllm.v1.attention.backends.mla.index_group import (
    HiSparseMLAIndexGroup,
    SparseMLAIndexGroupBuilder,
)
from vllm.v1.attention.backends.mla.indexer import (
    DeepseekV32IndexerMetadataBuilder,
)
from vllm.v1.attention.backends.mla.prefill import get_mla_prefill_backend
from vllm.v1.attention.backends.utils import (
    split_decodes_and_prefills,
    split_prefill_chunks,
)
from vllm.v1.attention.ops import flashmla
from vllm.v1.hisparse import runtime as hisparse_runtime
from vllm.v1.hisparse.runtime import (
    HiSparseCacheHandle,
    HiSparseRuntime,
    ResolvedHiSparseConfig,
    _has_hisparse_ops,
    build_hisparse_prefill_staging_plan,
    hisparse_prefill_staging_remap,
)
from vllm.v1.hisparse.types import SparseKVRowMirror

SPARSE_BACKEND_BATCH_SPECS = {
    name: BATCH_SPECS[name]
    for name in [
        "mixed_small",
        "mixed_medium",
        "small_prefill",
        "medium_prefill",
        "single_prefill",
    ]
}

SPARSE_BACKEND_BATCH_SPECS["large_q_prefill"] = BatchSpec(
    seq_lens=[1024] * 2, query_lens=[256] * 2
)
SPARSE_BACKEND_BATCH_SPECS["large_q_pure_prefill"] = BatchSpec(
    seq_lens=[256] * 2, query_lens=[256] * 2
)

DEVICE_TYPE = current_platform.device_type


def test_nope_flashinfer_sparse_mla_uses_model_scale(monkeypatch):
    """Weight absorption must not change the model's attention temperature."""
    model_scale = 256**-0.5
    kv_lora_rank = 512
    topk = torch.zeros((1, 1), dtype=torch.int32)
    metadata = SimpleNamespace(
        req_id_per_token=torch.zeros(1, dtype=torch.int32),
        block_table=torch.zeros((1, 1), dtype=torch.int32),
        block_size=1,
    )
    recorded_scale = None

    impl = object.__new__(FlashInferMLASparseImpl)
    impl.scale = model_scale
    impl.qk_nope_head_dim = 256
    impl.kv_lora_rank = kv_lora_rank
    impl.qk_rope_head_dim = 0
    impl.kv_cache_dtype = "auto"
    impl.topk_indices_buffer = topk
    impl.dcp_world_size = 1
    impl._workspace_buffer = torch.empty(1)
    impl.bmm1_scale = None
    impl.bmm2_scale = None
    impl.is_nope_mla = True
    impl.need_to_return_lse_for_decode = False
    impl.index_group = MagicMock()
    impl.index_group_index = 0
    impl.index_group.convert_logical_to_physical_topk.return_value = (
        topk,
        torch.ones(1, dtype=torch.int32),
    )

    import flashinfer.decode

    def fake_flashinfer(**kwargs):
        nonlocal recorded_scale
        recorded_scale = kwargs["bmm1_scale"]
        return torch.zeros((1, 1, 1, kv_lora_rank))

    monkeypatch.setattr(
        flashinfer.decode,
        "trtllm_batch_decode_with_kv_cache_mla",
        fake_flashinfer,
    )
    impl.forward_mqa(
        torch.zeros(1, 1, kv_lora_rank),
        torch.zeros(1, 1, kv_lora_rank),
        metadata,
        SimpleNamespace(),
    )

    assert recorded_scale == model_scale
    assert recorded_scale != kv_lora_rank**-0.5


def test_hisparse_routes_prefill_to_sparse_mqa():
    layer = SimpleNamespace(hisparse_cache=object())

    assert not mla_attention.MLAAttention._use_sparse_mha(layer, SimpleNamespace())


def test_hisparse_metadata_keeps_short_prefill_indexer_enabled():
    config = SimpleNamespace(
        attention_config=SimpleNamespace(
            hisparse_config=object(), sparse_mla_force_mqa=False
        )
    )

    assert not _use_dense_mha_prefill(config, prefill_max_seq_len=512, topk_tokens=2048)


def test_unified_mla_prepares_batch_without_slot_mapping(monkeypatch):
    layer = SimpleNamespace(
        prepare_kv_cache_update=MagicMock(),
        update_kv_cache=MagicMock(),
    )
    metadata = object()
    monkeypatch.setattr(
        mla_attention,
        "get_attention_context",
        lambda _: (metadata, layer, torch.empty(0), None),
    )
    tensor = torch.empty(0, device=DEVICE_TYPE)

    mla_attention.unified_mla_kv_cache_update(tensor, tensor, "layer", "auto", tensor)

    layer.prepare_kv_cache_update.assert_called_once_with(metadata)
    layer.update_kv_cache.assert_not_called()


def _float_to_e8m0_truncate(f: float) -> float:
    """Simulate SM100's float -> e8m0 -> bf16 scale conversion.
    e8m0 format only stores the exponent (power of 2).
    cudaRoundZero truncates toward zero, meaning we round down to the
    nearest power of 2.
    """
    if f <= 0:
        return 0.0
    # e8m0 = floor(log2(f)), then 2^(e8m0)
    # This is equivalent to truncating to the nearest power of 2 below f
    exp = math.floor(math.log2(f))
    return 2.0**exp


def _dequantize_fp8_ds_mla_entry(
    cache_slice: torch.Tensor,
    kv_lora_rank: int,
    rope_dim: int,
    dtype: torch.dtype,
    simulate_sm100_e8m0_scales: bool = False,
) -> tuple[torch.Tensor, torch.Tensor]:
    """Dequantize a single fp8_ds_mla cache entry back to latent + rope.

    Args:
        simulate_sm100_e8m0_scales: If True, simulate the SM100 kernel's
            float -> e8m0 -> bf16 scale conversion path.

    """
    # The first kv_lora_rank bytes store FP8 latent values with one scale per
    # 128 element tile written as float32 right after the latent payload.
    scales = cache_slice.view(torch.float32)[kv_lora_rank // 4 : kv_lora_rank // 4 + 4]
    latent = torch.empty(kv_lora_rank, dtype=torch.float16, device=cache_slice.device)
    for tile_idx in range(4):
        tile_start = tile_idx * 128
        tile_end = tile_start + 128
        scale_val = float(scales[tile_idx].item())
        if simulate_sm100_e8m0_scales:
            # Simulate the lossy float -> e8m0 -> bf16 conversion
            scale_val = _float_to_e8m0_truncate(scale_val)
        ops.convert_fp8(
            latent[tile_start:tile_end],
            cache_slice[tile_start:tile_end],
            scale_val,
            kv_dtype="fp8",
        )
    latent = latent.to(dtype)

    rope_offset = kv_lora_rank // 2 + 8
    rope_vals = cache_slice.view(dtype)[rope_offset : rope_offset + rope_dim]
    return latent, rope_vals.clone()


def _quantize_dequantize_fp8_ds_mla(
    kv_c: torch.Tensor,
    k_pe: torch.Tensor,
    block_size: int,
    scale: torch.Tensor,
    simulate_sm100_e8m0_scales: bool = False,
) -> tuple[torch.Tensor, torch.Tensor]:
    """Round-trip kv_c/k_pe though the fp8_ds_mla cache layout.

    Args:
        simulate_sm100_e8m0_scales: If True, simulate the SM100 kernel's
            float -> e8m0 -> bf16 scale conversion in dequantization.

    """
    if kv_c.numel() == 0:
        return kv_c.clone(), k_pe.clone()

    kv_lora_rank = kv_c.shape[-1]
    rope_dim = k_pe.shape[-1]
    num_tokens = kv_c.shape[0]
    num_blocks = max(1, math.ceil(num_tokens / block_size))
    entry_size = kv_lora_rank + 4 * 4 + 2 * rope_dim

    tmp_cache = torch.zeros(
        num_blocks, block_size, entry_size, dtype=torch.uint8, device=kv_c.device
    )
    slot_mapping = torch.arange(num_tokens, dtype=torch.long, device=kv_c.device)

    ops.concat_and_cache_mla(
        kv_c, k_pe, tmp_cache, slot_mapping, kv_cache_dtype="fp8_ds_mla", scale=scale
    )

    dequant_kv_c = torch.empty_like(kv_c)
    dequant_k_pe = torch.empty_like(k_pe)

    for token_idx in range(num_tokens):
        slot = slot_mapping[token_idx].item()
        block_idx = slot // block_size
        block_offset = slot % block_size
        cache_slice = tmp_cache[block_idx, block_offset]
        latent, rope_vals = _dequantize_fp8_ds_mla_entry(
            cache_slice,
            kv_lora_rank,
            rope_dim,
            kv_c.dtype,
            simulate_sm100_e8m0_scales=simulate_sm100_e8m0_scales,
        )
        dequant_kv_c[token_idx] = latent
        dequant_k_pe[token_idx] = rope_vals

    return dequant_kv_c, dequant_k_pe


_E2M1_TABLE = [0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0]


def _quantize_dequantize_nvfp4_ds_mla(
    kv_c: torch.Tensor,
    k_pe: torch.Tensor,
    block_size: int,
    kv_cache_dtype: str,
    scale: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
    """Round-trip kv_c/k_pe through the nvfp4_ds_mla cache layout.

    Layout (per token, uint8): [e2m1-packed NoPE | unscaled e4m3 RoPE |
    per-16 e4m3 NoPE SFs]. The SF bytes are stored permuted (an 8x4 -> 4x8
    transpose: the scale for element block s lives at byte 8 * (s & 3) +
    (s >> 2)) so that the 8 scales one FlashMLA dequant thread needs are
    contiguous. Dequant: x = value * float(sf), which is exact in bf16, so
    this python dequant matches the kernels bit-for-bit.
    """
    if kv_c.numel() == 0:
        return kv_c.clone(), k_pe.clone()

    kv_lora_rank = kv_c.shape[-1]
    rope_dim = k_pe.shape[-1]
    num_tokens = kv_c.shape[0]
    num_blocks = max(1, math.ceil(num_tokens / block_size))
    entry_size = (
        math.ceil((kv_lora_rank // 2 + rope_dim + kv_lora_rank // 16) / 16) * 16
    )
    sf_nope_off = kv_lora_rank // 2 + rope_dim

    tmp_cache = torch.zeros(
        num_blocks, block_size, entry_size, dtype=torch.uint8, device=kv_c.device
    )
    slot_mapping = torch.arange(num_tokens, dtype=torch.long, device=kv_c.device)
    ops.concat_and_cache_mla(
        kv_c, k_pe, tmp_cache, slot_mapping, kv_cache_dtype=kv_cache_dtype, scale=scale
    )

    tokens = tmp_cache.view(-1, entry_size)[:num_tokens]
    table = torch.tensor(
        _E2M1_TABLE + [-v for v in _E2M1_TABLE],
        dtype=torch.float32,
        device=kv_c.device,
    )

    def unpack_e2m1(packed: torch.Tensor) -> torch.Tensor:
        lo = (packed & 0xF).long()
        hi = (packed >> 4).long()
        return table[torch.stack([lo, hi], dim=-1).flatten(-2)]

    nope_vals = unpack_e2m1(tokens[:, : kv_lora_rank // 2])
    num_nope_sf = kv_lora_rank // 16
    nope_sf = (
        # undo the on-wire SF permutation -> element-block order
        tokens[:, sf_nope_off : sf_nope_off + num_nope_sf]
        .unflatten(-1, (4, num_nope_sf // 4))
        .transpose(-1, -2)
        .flatten(-2)
        .view(torch.float8_e4m3fn)
        .float()
    )
    dequant_kv_c = (
        (nope_vals.unflatten(-1, (-1, 16)) * nope_sf.unsqueeze(-1)).flatten(-2)
    ).to(kv_c.dtype)

    # RoPE: plain e4m3, no scale factor
    rope_raw = tokens[:, kv_lora_rank // 2 : sf_nope_off]
    dequant_k_pe = rope_raw.view(torch.float8_e4m3fn).to(k_pe.dtype)

    return dequant_kv_c, dequant_k_pe


@pytest.mark.parametrize(
    "backend_cls",
    [FlashMLASparseBackend, FlashInferMLASparseTRTLLMBackend],
    ids=["FlashMLA", "FlashInferTRTLLM"],
)
@pytest.mark.parametrize("batch_name", list(SPARSE_BACKEND_BATCH_SPECS.keys()))
@pytest.mark.parametrize(
    "kv_cache_dtype",
    ["auto", "fp8", "fp8_ds_mla", "nvfp4_ds_mla"],
)
@pytest.mark.parametrize("tensor_parallel_size", [1, 2, 4])
@pytest.mark.parametrize("block_size", [32, 64])
@pytest.mark.parametrize(("q_scale", "k_scale"), [(1.0, 1.0), (2.0, 3.0)])
def test_sparse_backend_decode_correctness(
    default_vllm_config,
    dist_init,
    backend_cls,
    batch_name,
    kv_cache_dtype,
    tensor_parallel_size,
    block_size,
    workspace_init,
    q_scale: float,
    k_scale: float,
    monkeypatch,
):
    if (
        batch_name == "large_q_pure_prefill"
        and backend_cls == FlashMLASparseBackend
        and kv_cache_dtype == "fp8_ds_mla"
        and tensor_parallel_size == 4
    ):
        monkeypatch.setattr(
            "vllm.v1.attention.backends.mla.flashmla_sparse.split_prefill_chunks",
            lambda rows, capacity: [(i, i + 1) for i in range(len(rows))],
        )
    if kv_cache_dtype not in backend_cls.supported_kv_cache_dtypes:
        pytest.skip(f"{backend_cls.get_name()} does not support {kv_cache_dtype}")

    if (
        backend_cls == FlashMLASparseBackend
        and kv_cache_dtype.startswith("fp8")
        and kv_cache_dtype != "fp8_ds_mla"
    ):
        pytest.skip(
            "FlashMLA Sparse Attention backend fp8 only supports "
            "fp8_ds_mla kv-cache dtype"
        )

    if kv_cache_dtype == "nvfp4_ds_mla":
        # Unlike "fp8" (an alias canonicalized to "fp8_ds_mla" above), the
        # NVFP4 DS-MLA dtype must reach the backend unchanged.
        assert (
            _canonicalize_sparse_mla_kv_cache_dtype(backend_cls, kv_cache_dtype)
            == kv_cache_dtype
        )
        device_capability = current_platform.get_device_capability()
        if device_capability is None or device_capability.major != 10:
            pytest.skip("The NVFP4 DS-MLA kv-cache dtype requires SM 10.x")

    supported_block_sizes = backend_cls.get_supported_kernel_block_sizes()
    if block_size not in supported_block_sizes:
        pytest.skip(
            f"{backend_cls.get_name()} does not support block_size={block_size}"
        )

    if backend_cls == FlashMLASparseBackend:
        ok, reason = flashmla.is_flashmla_sparse_supported()
        if not ok:
            pytest.skip(reason)
    elif backend_cls == FlashInferMLASparseTRTLLMBackend:
        device_capability = current_platform.get_device_capability()
        if device_capability is None or not backend_cls.supports_compute_capability(
            device_capability
        ):
            pytest.skip("FlashInferMLASparseTRTLLMBackend requires SM 10.x capability")

    batch_spec = SPARSE_BACKEND_BATCH_SPECS[batch_name]
    use_fp8_ds_mla_quantization = kv_cache_dtype == "fp8_ds_mla"
    use_nvfp4_ds_mla_quantization = kv_cache_dtype == "nvfp4_ds_mla"

    device = torch.device(DEVICE_TYPE)
    dtype = torch.bfloat16

    # Model hyper-parameters (kept intentionally small for the unit test)
    total_num_heads = 128
    # Compute per-rank heads for simulated TP
    num_heads = max(1, total_num_heads // tensor_parallel_size)

    kv_lora_rank = 512
    qk_nope_head_dim = 128
    qk_rope_head_dim = 64
    v_head_dim = 128
    head_size = kv_lora_rank + qk_rope_head_dim
    topk_tokens = 128

    max_seqlen = max(batch_spec.seq_lens)
    total_cache_tokens = sum(batch_spec.seq_lens)

    # Note: We use TP=1 to avoid multi-GPU requirements in CI.
    # The test simulates head partitioning via mocked methods below.
    vllm_config = create_vllm_config(
        model_name="deepseek-ai/DeepSeek-V2-Lite-Chat",
        tensor_parallel_size=1,
        max_model_len=max_seqlen,
        num_gpu_blocks=max(2048, cdiv(total_cache_tokens, block_size) + 1),
        block_size=block_size,
        hf_config_override={
            "index_topk": topk_tokens,
            "attn_module_list_cfg": [{"topk_tokens": topk_tokens}],
        },
    )
    model_config = vllm_config.model_config
    model_config.hf_text_config = SimpleNamespace(
        index_topk=topk_tokens,
        q_lora_rank=None,
        kv_lora_rank=kv_lora_rank,
        qk_nope_head_dim=qk_nope_head_dim,
        qk_rope_head_dim=qk_rope_head_dim,
        v_head_dim=v_head_dim,
        model_type="deepseek_v2",
    )
    del model_config.hf_config.index_topk  # Composite configs only nest this field.
    model_config.dtype = dtype
    model_config.get_num_attention_heads = MethodType(
        lambda self, parallel_config: num_heads,
        model_config,
    )
    model_config.get_num_kv_heads = MethodType(
        lambda self, parallel_config: 1, model_config
    )
    model_config.get_head_size = MethodType(lambda self: head_size, model_config)
    model_config.get_sliding_window = MethodType(lambda self: None, model_config)

    kv_cache_spec = create_standard_kv_cache_spec(vllm_config)

    torch.manual_seed(0)

    scale = 1.0 / math.sqrt(head_size)

    # Shared MLA projection weights to keep reference and backend in sync
    W_UK = torch.rand(
        kv_lora_rank, num_heads, qk_nope_head_dim, dtype=dtype, device=device
    )
    W_UV = torch.rand(kv_lora_rank, num_heads, v_head_dim, dtype=dtype, device=device)

    # Build synthetic decode-only workload
    seq_lens = batch_spec.seq_lens
    query_lens = batch_spec.query_lens

    # Pre-compute positions and sparse indices for all tokens.
    # We need these BEFORE computing the reference to use sparse attention masks.
    total_query_tokens = sum(query_lens)
    positions = []
    for i in range(batch_spec.batch_size):
        s_len = seq_lens[i]
        q_len = query_lens[i]
        ctx_len = s_len - q_len
        for q_idx in range(q_len):
            positions.append(ctx_len + q_idx)

    # Create sparse indices with UNIQUE per-token offsets to catch bugs where
    # the kernel uses wrong indices for some tokens (e.g., due to incorrect
    # tensor shapes like [1, num_tokens, ...] instead of [num_tokens, 1, ...]).
    # Also include -1 masked indices to verify the kernel handles them correctly.
    sparse_indices = torch.empty(
        total_query_tokens, topk_tokens, dtype=torch.int32, device=device
    )
    for tok_idx in range(total_query_tokens):
        max_valid_idx = positions[tok_idx]
        offset = tok_idx * 7  # Prime number for varied offsets
        # Use only half the topk indices as valid, mask the rest with -1
        # This tests that the kernel correctly ignores -1 indices
        num_valid = min(topk_tokens // 2, max_valid_idx + 1)
        if num_valid > 0:
            valid_range = torch.arange(num_valid, device=device, dtype=torch.int32)
            tok_indices = (valid_range + offset) % (max_valid_idx + 1)
            # Pad with -1 for the remaining positions
            tok_indices = torch.cat(
                [
                    tok_indices,
                    torch.full(
                        (topk_tokens - num_valid,), -1, device=device, dtype=torch.int32
                    ),
                ]
            )
        else:
            tok_indices = torch.full(
                (topk_tokens,), -1, device=device, dtype=torch.int32
            )
            tok_indices[0] = 0  # At least one valid index
        sparse_indices[tok_idx] = tok_indices

    all_q_vllm, all_kv_c_vllm, all_k_pe_vllm = [], [], []
    kv_c_contexts, k_pe_contexts = [], []
    reference_outputs = []

    kv_cache_scale = torch.tensor(k_scale, dtype=torch.float32, device=device)
    global_token_idx = 0

    for i in range(batch_spec.batch_size):
        s_len = seq_lens[i]
        q_len = query_lens[i]
        ctx_len = s_len - q_len

        q_c = torch.rand(
            q_len,
            num_heads,
            qk_nope_head_dim + qk_rope_head_dim,
            dtype=dtype,
            device=device,
        )
        kv_c_full = torch.rand(s_len, kv_lora_rank, dtype=dtype, device=device)
        k_pe_full = torch.rand(s_len, 1, qk_rope_head_dim, dtype=dtype, device=device)

        if use_fp8_ds_mla_quantization:
            is_sm100 = torch.cuda.get_device_capability()[0] >= 10
            kv_c_full, k_pe_squeezed = _quantize_dequantize_fp8_ds_mla(
                kv_c_full,
                k_pe_full.squeeze(1),
                block_size=block_size,
                scale=kv_cache_scale,
                simulate_sm100_e8m0_scales=is_sm100,
            )
            k_pe_full = k_pe_squeezed.unsqueeze(1)
        elif use_nvfp4_ds_mla_quantization:
            kv_c_full, k_pe_squeezed = _quantize_dequantize_nvfp4_ds_mla(
                kv_c_full,
                k_pe_full.squeeze(1),
                block_size=block_size,
                kv_cache_dtype=kv_cache_dtype,
                scale=kv_cache_scale,
            )
            k_pe_full = k_pe_squeezed.unsqueeze(1)

        q_nope, q_pe = q_c.split([qk_nope_head_dim, qk_rope_head_dim], dim=-1)
        ql_nope = torch.einsum("qnh,lnh->qnl", q_nope, W_UK)
        q_mqa = torch.cat([ql_nope, q_pe], dim=-1)

        k_mqa = torch.cat([kv_c_full, k_pe_full.squeeze(1)], dim=-1)
        v_mqa = kv_c_full

        # Compute sparse SDPA reference per query token using its sparse indices
        for q_idx in range(q_len):
            tok_sparse_idx = sparse_indices[global_token_idx]
            valid_mask = tok_sparse_idx >= 0
            valid_indices = tok_sparse_idx[valid_mask].long()

            q_tok = q_mqa[q_idx : q_idx + 1]  # [1, num_heads, head_dim]
            k_sparse = k_mqa[valid_indices]  # [num_valid, head_dim]
            v_sparse = v_mqa[valid_indices]  # [num_valid, kv_lora_rank]

            k_sparse = k_sparse.unsqueeze(1).expand(-1, num_heads, -1)
            v_sparse = v_sparse.unsqueeze(1).expand(-1, num_heads, -1)

            # SDPA: [1, num_heads, 1, head_dim] x [1, num_heads, num_valid, head_dim]
            q_sdpa_in = q_tok.unsqueeze(0).transpose(1, 2)
            k_sdpa_in = k_sparse.unsqueeze(0).transpose(1, 2)
            v_sdpa_in = v_sparse.unsqueeze(0).transpose(1, 2)

            sdpa_out = torch.nn.functional.scaled_dot_product_attention(
                q_sdpa_in, k_sdpa_in, v_sdpa_in, scale=scale
            )
            sdpa_out = sdpa_out.transpose(1, 2).squeeze(
                0
            )  # [1, num_heads, kv_lora_rank]

            sdpa_out = torch.einsum("qnl,lnv->qnv", sdpa_out, W_UV)
            reference_outputs.append(sdpa_out.flatten(start_dim=-2))

            global_token_idx += 1

        all_q_vllm.append(q_c)
        all_kv_c_vllm.append(kv_c_full[ctx_len:])
        all_k_pe_vllm.append(k_pe_full[ctx_len:])
        kv_c_contexts.append(kv_c_full[: ctx_len + 1])
        k_pe_contexts.append(k_pe_full[: ctx_len + 1])

    query_vllm = torch.cat(all_q_vllm, dim=0)
    kv_c_vllm = torch.cat(all_kv_c_vllm, dim=0)
    k_pe_vllm = torch.cat(all_k_pe_vllm, dim=0)
    sdpa_reference = torch.cat(reference_outputs, dim=0)

    vllm_config.cache_config.cache_dtype = kv_cache_dtype
    vllm_config.model_config.hf_text_config.index_topk = topk_tokens

    common_attn_metadata = create_common_attn_metadata(
        batch_spec,
        vllm_config.cache_config.block_size,
        device,
        arange_block_indices=True,
    )

    kv_cache = create_and_prepopulate_kv_cache(
        kv_c_contexts=kv_c_contexts,
        k_pe_contexts=k_pe_contexts,
        block_size=vllm_config.cache_config.block_size,
        head_size=head_size,
        dtype=dtype,
        device=device,
        num_blocks=vllm_config.cache_config.num_gpu_blocks,
        common_attn_metadata=common_attn_metadata,
        randomize_blocks=False,
        kv_cache_dtype=kv_cache_dtype,
        scale=kv_cache_scale,
    )

    # The sparse builder clones the layer's dense-MHA prefill backend from
    # static_forward_context; register a mock layer carrying one.
    prefill_backend = get_mla_prefill_backend(vllm_config)(
        num_heads=num_heads,
        scale=scale,
        kv_lora_rank=kv_lora_rank,
        qk_nope_head_dim=qk_nope_head_dim,
        qk_rope_head_dim=qk_rope_head_dim,
        v_head_dim=v_head_dim,
        vllm_config=vllm_config,
    )
    vllm_config.compilation_config.static_forward_context["placeholder"] = (
        SimpleNamespace(prefill_backend=prefill_backend)
    )

    builder_cls = backend_cls.get_builder_cls()
    builder = builder_cls(kv_cache_spec, ["placeholder"], vllm_config, device)
    metadata = builder.build(
        common_prefix_len=0, common_attn_metadata=common_attn_metadata
    )

    # Use the pre-computed sparse_indices for the mock indexer
    mock_indexer = SimpleNamespace(topk_indices_buffer=sparse_indices)

    kv_b_proj_weight = torch.cat([W_UK, W_UV], dim=-1)
    kv_b_proj_weight = kv_b_proj_weight.view(
        kv_lora_rank, num_heads * (qk_nope_head_dim + v_head_dim)
    )

    mock_kv_b_proj = ColumnParallelLinear(
        input_size=kv_lora_rank,
        output_size=num_heads * (qk_nope_head_dim + v_head_dim),
        bias=False,
    ).to(device=device, dtype=dtype)
    mock_kv_b_proj.weight = torch.nn.Parameter(kv_b_proj_weight.T.contiguous())

    impl_cls = backend_cls.get_impl_cls()
    with set_current_vllm_config(vllm_config):
        impl = impl_cls(
            num_heads=num_heads,
            head_size=head_size,
            scale=scale,
            num_kv_heads=1,
            alibi_slopes=None,
            sliding_window=None,
            kv_cache_dtype=vllm_config.cache_config.cache_dtype,
            logits_soft_cap=None,
            attn_type="decoder",
            kv_sharing_target_layer_name=None,
            q_lora_rank=None,
            kv_lora_rank=kv_lora_rank,
            qk_nope_head_dim=qk_nope_head_dim,
            qk_rope_head_dim=qk_rope_head_dim,
            qk_head_dim=qk_nope_head_dim + qk_rope_head_dim,
            v_head_dim=v_head_dim,
            kv_b_proj=mock_kv_b_proj,
            indexer=mock_indexer,
        )

        impl.process_weights_after_loading(dtype)

        # Create mock sparse MLA layer with weight matrices
        mock_layer = MockSparseMLAAttentionLayer(
            impl=impl,
            num_heads=num_heads,
            qk_nope_head_dim=qk_nope_head_dim,
            qk_rope_head_dim=qk_rope_head_dim,
            v_head_dim=v_head_dim,
            kv_lora_rank=kv_lora_rank,
            device=device,
            W_UK=W_UK,
            W_UV=W_UV,
            q_scale=q_scale,
            k_scale=k_scale,
        )

    out_buffer = torch.empty(
        metadata.num_actual_tokens, num_heads * v_head_dim, dtype=dtype, device=device
    )

    with torch.inference_mode():
        if backend_cls == FlashMLASparseBackend and kv_cache_dtype == "fp8_ds_mla":
            from vllm.v1.worker.workspace import current_workspace_manager

            current_workspace_manager().lock()
        backend_output = mock_layer.forward_impl(
            query_vllm,
            kv_c_vllm,
            k_pe_vllm,
            kv_cache,
            metadata,
            out_buffer,
        )

    assert backend_output.shape == sdpa_reference.shape
    assert backend_output.dtype == sdpa_reference.dtype
    assert torch.isfinite(backend_output).all()

    # FP8 quantization introduces some error, but should be within reasonable bounds
    # BF16 (auto) should be very accurate, FP8 allows slightly more tolerance
    if kv_cache_dtype.startswith("fp8"):
        torch.testing.assert_close(
            backend_output, sdpa_reference, rtol=0.065, atol=0.05
        )
    else:
        torch.testing.assert_close(backend_output, sdpa_reference, rtol=0.01, atol=0.01)


def _triton_convert_reference_impl(
    req_ids: torch.Tensor,
    block_table: torch.Tensor,
    token_indices: torch.Tensor,
    block_size: int,
    num_topk_tokens: int,
    HAS_PREFILL_WORKSPACE: bool = False,
    prefill_workspace_request_ids: torch.Tensor | None = None,
    prefill_workspace_starts: torch.Tensor | None = None,
) -> torch.Tensor:
    """Reference implementation for triton_convert_req_index_to_global_index."""
    num_tokens = req_ids.shape[0]
    max_blocks_per_req = block_table.shape[1]
    result = torch.empty(
        num_tokens, num_topk_tokens, dtype=torch.int32, device=req_ids.device
    )

    for token_id in range(num_tokens):
        req_id = req_ids[token_id].item()

        # Determine if this token uses workspace or paged cache
        use_prefill_workspace = False
        workspace_start = 0
        if HAS_PREFILL_WORKSPACE and prefill_workspace_request_ids is not None:
            assert prefill_workspace_starts is not None
            prefill_req_id = prefill_workspace_request_ids[token_id].item()
            if prefill_req_id >= 0:
                use_prefill_workspace = True
                workspace_start = prefill_workspace_starts[prefill_req_id].item()

        for idx_id in range(num_topk_tokens):
            token_idx = token_indices[token_id, idx_id].item()

            if token_idx == -1:
                result[token_id, idx_id] = -1
            elif use_prefill_workspace:
                # Prefill + using prefill workspace: map to workspace offset
                result[token_id, idx_id] = workspace_start + token_idx
            else:
                # Decode: map to paged cache
                block_id = token_idx // block_size
                if block_id >= max_blocks_per_req:
                    result[token_id, idx_id] = -1
                else:
                    block_num = block_table[req_id, block_id].item()
                    offset = token_idx % block_size
                    result[token_id, idx_id] = block_num * block_size + offset

    return result


@pytest.mark.parametrize("block_size", [16, 64, 128])
@pytest.mark.parametrize("num_topk_tokens", [128, 256, 512])
@pytest.mark.skipif(
    torch.cuda.get_device_capability() < (9, 0),
    reason="FlashMLASparseBackend requires CUDA 9.0 or higher",
)
def test_triton_convert_req_index_to_global_index_decode_only(
    block_size, num_topk_tokens
):
    device = torch.device(DEVICE_TYPE)
    num_tokens = 8
    num_requests = 4
    max_blocks_per_req = 10

    req_id = torch.randint(
        0, num_requests, (num_tokens,), dtype=torch.int32, device=device
    )
    block_table = torch.randint(
        0, 100, (num_requests, max_blocks_per_req), dtype=torch.int32, device=device
    )

    token_indices = torch.randint(
        0,
        block_size * max_blocks_per_req,
        (num_tokens, num_topk_tokens),
        dtype=torch.int32,
        device=device,
    )

    # Set some to -1 to test masking
    token_indices[0, :10] = -1
    token_indices[3, 50:60] = -1

    # Set some to out of bounds
    token_indices[2, 100:110] = max_blocks_per_req * block_size
    token_indices[6, 150:160] = max_blocks_per_req * block_size

    result = triton_convert_req_index_to_global_index(
        req_id,
        block_table,
        token_indices,
        BLOCK_SIZE=block_size,
        NUM_TOPK_TOKENS=num_topk_tokens,
    )

    reference_result = _triton_convert_reference_impl(
        req_id,
        block_table,
        token_indices,
        block_size,
        num_topk_tokens,
    )

    torch.testing.assert_close(result, reference_result, rtol=0, atol=0)


def test_index_group_convert_during_piecewise_capture():
    """In piecewise cudagraph mode the indexer is captured while the convert
    runs in the following eager break. The convert's side stream must not
    wait on ``logical_topk_ready`` there: an event recorded inside a captured
    segment is graph-local, and an eager wait on it raises
    cudaErrorInvalidValue."""
    device = torch.device(DEVICE_TYPE)
    num_tokens, num_topk, num_requests, blocks_per_req, block_size = 8, 128, 4, 4, 16

    logical_topk_indices = torch.randint(
        0,
        block_size * blocks_per_req,
        (num_tokens, num_topk),
        dtype=torch.int32,
        device=device,
    )
    builder = SparseMLAIndexGroupBuilder(logical_topk_indices)
    group, layer_index = builder.register_layer(is_index_producing_layer=True)
    assert layer_index == 0 and group.has_indexer

    req_id = torch.arange(num_tokens, dtype=torch.int32, device=device) % num_requests
    block_table = torch.arange(
        num_requests * blocks_per_req, dtype=torch.int32, device=device
    ).view(num_requests, blocks_per_req)
    attn_metadata = SimpleNamespace(
        req_id_per_token=req_id, block_table=block_table, block_size=block_size
    )
    expected = triton_convert_req_index_to_global_index(
        req_id,
        block_table,
        logical_topk_indices,
        BLOCK_SIZE=block_size,
        NUM_TOPK_TOKENS=num_topk,
    )

    vllm_config = create_vllm_config(model_name="Qwen/Qwen3.5-0.8B")
    marker = torch.zeros(1, device=device)
    capture_stream = torch.cuda.Stream(device=device)
    # torch.cuda.stream routes through vLLM's patched torch.cuda.set_stream,
    # so current_stream() tracks the capture stream as in production.
    with (
        torch.cuda.stream(capture_stream),
        set_forward_context(
            None, vllm_config, cudagraph_runtime_mode=CUDAGraphMode.PIECEWISE
        ),
    ):
        capture = BreakableCUDAGraphCapture()
        with capture:
            marker.add_(1)
            # Record inside a captured segment, convert in the eager
            # break — mirrors mla.py -> unified_mla_attention_with_output.
            group.set_logical_topk_ready(layer_index)
            capture.add_eager(
                lambda: group.convert_logical_to_physical_topk(
                    layer_index,
                    logical_topk_indices,
                    attn_metadata,
                    block_stride_rows=None,
                    return_valid_counts=False,
                )
            )
            marker.add_(1)
        capture.replay()
        capture_stream.synchronize()

    result = group.physical_topk_indices[:num_tokens]
    torch.testing.assert_close(result, expected, rtol=0, atol=0)


@pytest.mark.parametrize("block_size", [16])
@pytest.mark.skipif(
    torch.cuda.get_device_capability() < (9, 0),
    reason="FlashMLASparseBackend requires CUDA 9.0 or higher",
)
def test_triton_convert_req_index_to_global_index_with_prefill_workspace(block_size):
    device = torch.device(DEVICE_TYPE)
    num_requests = 4
    max_blocks_per_req = 8
    num_topk_tokens = 128

    # First 6 tokens are decode (reqs 0, 1), last 6 are prefill (reqs 2, 3)
    req_id = torch.tensor(
        [0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3], dtype=torch.int32, device=device
    )
    prefill_workspace_request_ids = torch.tensor(
        [-1, -1, -1, -1, -1, -1, 0, 0, 0, 1, 1, 1], dtype=torch.int32, device=device
    )

    # Workspace starts for the 2 prefill reqs: req 2 starts at 0, req 3 starts at 100
    prefill_workspace_starts = torch.tensor([0, 100], dtype=torch.int32, device=device)

    block_table = torch.randint(
        0, 50, (num_requests, max_blocks_per_req), dtype=torch.int32, device=device
    )
    token_indices = torch.randint(
        0,
        block_size * max_blocks_per_req,
        (req_id.shape[0], num_topk_tokens),
        dtype=torch.int32,
        device=device,
    )

    # Set some to -1 to test masking
    token_indices[0, :10] = -1
    token_indices[3, 50:60] = -1

    # Set some to out of bounds
    token_indices[2, 100:110] = max_blocks_per_req * block_size
    token_indices[6, 150:160] = max_blocks_per_req * block_size

    result = triton_convert_req_index_to_global_index(
        req_id,
        block_table,
        token_indices,
        BLOCK_SIZE=block_size,
        NUM_TOPK_TOKENS=num_topk_tokens,
        HAS_PREFILL_WORKSPACE=True,
        prefill_workspace_request_ids=prefill_workspace_request_ids,
        prefill_workspace_starts=prefill_workspace_starts,
    )

    reference_result = _triton_convert_reference_impl(
        req_id,
        block_table,
        token_indices,
        block_size,
        num_topk_tokens,
        HAS_PREFILL_WORKSPACE=True,
        prefill_workspace_request_ids=prefill_workspace_request_ids,
        prefill_workspace_starts=prefill_workspace_starts,
    )

    torch.testing.assert_close(result, reference_result, rtol=0, atol=0)


@pytest.mark.skipif(
    torch.cuda.get_device_capability() < (9, 0),
    reason="FlashMLASparseBackend requires CUDA 9.0 or higher",
)
def test_triton_convert_rejects_req_id_longer_than_token_indices():
    """Guard against the #47327 regression: the kernel grid is sized by
    req_id but the output is allocated like token_indices, so a full-batch
    req_id combined with an MQA-subset token_indices wrote past the end of
    the output buffer. The wrapper must reject the length mismatch instead
    of corrupting memory."""
    device = torch.device(DEVICE_TYPE)
    num_topk_tokens = 128
    block_size = 64
    block_table = torch.arange(40, dtype=torch.int32, device=device).view(4, 10)

    # Full batch: 2 decode tokens + 10 prefill tokens
    req_id_full = torch.tensor(
        [0, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3], dtype=torch.int32, device=device
    )
    num_mqa_tokens = 2
    token_indices = torch.randint(
        0,
        block_size * 10,
        (num_mqa_tokens, num_topk_tokens),
        dtype=torch.int32,
        device=device,
    )

    with pytest.raises(AssertionError, match="must cover the same tokens"):
        triton_convert_req_index_to_global_index(
            req_id_full,
            block_table,
            token_indices,
            BLOCK_SIZE=block_size,
            NUM_TOPK_TOKENS=num_topk_tokens,
        )

    # The sliced call is the intended usage and must match the reference.
    result = triton_convert_req_index_to_global_index(
        req_id_full[:num_mqa_tokens],
        block_table,
        token_indices,
        BLOCK_SIZE=block_size,
        NUM_TOPK_TOKENS=num_topk_tokens,
    )
    reference = _triton_convert_reference_impl(
        req_id_full[:num_mqa_tokens],
        block_table,
        token_indices,
        block_size,
        num_topk_tokens,
    )
    torch.testing.assert_close(result, reference, rtol=0, atol=0)


@pytest.mark.skipif(
    torch.cuda.get_device_capability() < (9, 0),
    reason="FlashMLASparseBackend requires CUDA 9.0 or higher",
)
def test_flashmla_forward_bf16_kv_slices_req_id_to_mqa_tokens():
    """Guard against the #47327 regression: when the dense-MHA prefill split
    is active, forward_mqa only receives the leading decode tokens, but
    _forward_bf16_kv passed the full-batch req_id_per_token to the index
    conversion, making it write past the end of its output buffer. The call
    site must slice req_id_per_token to the MQA tokens."""
    device = torch.device(DEVICE_TYPE)
    num_topk_tokens = 128
    block_size = 64
    num_batch_tokens = 12
    num_mqa_tokens = 2

    attn_metadata = SimpleNamespace(
        req_id_per_token=torch.tensor(
            [0, 1] + [2] * 5 + [3] * 5, dtype=torch.int32, device=device
        ),
        block_table=torch.arange(40, dtype=torch.int32, device=device).view(4, 10),
        block_size=block_size,
        num_decode_tokens=num_mqa_tokens,
        num_actual_tokens=num_mqa_tokens,
    )
    assert attn_metadata.req_id_per_token.shape[0] == num_batch_tokens

    q = torch.zeros(num_mqa_tokens, 4, 576, dtype=torch.bfloat16, device=device)
    kv_cache = torch.zeros(40, block_size, 576, dtype=torch.bfloat16, device=device)
    topk_indices = torch.randint(
        0,
        block_size * 10,
        (num_mqa_tokens, num_topk_tokens),
        dtype=torch.int32,
        device=device,
    )

    captured = {}

    def _stub_kernel(q, kv, indices, lengths, actual_num_heads):
        captured["indices"] = indices
        captured["actual_num_heads"] = actual_num_heads
        return (
            torch.zeros(q.shape[0], q.shape[1], 512, dtype=q.dtype, device=q.device),
            None,
        )

    def _convert_topk(indices, metadata, **kwargs):
        return triton_convert_req_index_to_global_index(
            metadata.req_id_per_token[: indices.shape[0]],
            metadata.block_table,
            indices,
            BLOCK_SIZE=metadata.block_size,
            BLOCK_STRIDE_ROWS=kwargs["block_stride_rows"],
            NUM_TOPK_TOKENS=indices.shape[1],
            return_valid_counts=kwargs["return_valid_counts"],
        )

    stub_impl = SimpleNamespace(
        _bf16_flash_mla_kernel=_stub_kernel,
        _convert_logical_to_physical_topk=_convert_topk,
        index_group=None,
        index_group_index=0,
        pcp_dcp_kv_gather=False,
    )

    out, _ = FlashMLASparseImpl._forward_bf16_kv(
        stub_impl, q, kv_cache, topk_indices, attn_metadata, q.shape[1]
    )

    assert out.shape[0] == num_mqa_tokens
    assert captured["indices"].shape[0] == num_mqa_tokens
    assert captured["actual_num_heads"] == q.shape[1]
    reference = _triton_convert_reference_impl(
        attn_metadata.req_id_per_token[:num_mqa_tokens],
        attn_metadata.block_table,
        topk_indices,
        block_size,
        num_topk_tokens,
    )
    torch.testing.assert_close(captured["indices"], reference, rtol=0, atol=0)


@pytest.mark.parametrize(
    "seq_lens,max_buf,expected",
    [
        # Basic split: totals per chunk ≤ max_buf
        (torch.tensor([2, 3, 4, 2]), 5, [(0, 2), (2, 3), (3, 4)]),
        # Exact fits should split between items when adding the next would overflow
        (torch.tensor([5, 5, 5]), 5, [(0, 1), (1, 2), (2, 3)]),
        # All requests fit in a single chunk
        (torch.tensor([1, 1, 1]), 10, [(0, 3)]),
        # Large buffer
        (torch.tensor([4, 4, 4]), 100, [(0, 3)]),
    ],
)
def test_split_prefill_chunks(seq_lens, max_buf, expected):
    out = split_prefill_chunks(seq_lens, max_buf)
    assert out == expected


@pytest.mark.parametrize(
    ("max_query_len", "expected"),
    [(32768, True), (33024, False)],
)
def test_masked_mha_workspace_fits_single_request_boundary(max_query_len, expected):
    """A 32K prefill needs the default workspace exactly; shrinking it would
    push a supported request onto MQA."""
    assert (
        _masked_mha_workspace_fits(
            batch_size=1,
            max_query_len=max_query_len,
            max_context_chunk_seq_len=0,
            workspace_numel=GLOBAL_TOPK_MASK_MAX_BYTES // torch.int32.itemsize,
        )
        is expected
    )


@pytest.mark.parametrize(
    ("backend_name", "tensor_parallel_size", "query_len"),
    [
        ("FLASHMLA_SPARSE", 4, 48 * 1024),
        ("FLASHMLA_SPARSE", 8, 112 * 1024),
        ("FLASHINFER_MLA_SPARSE", 4, 36 * 1024),
        ("FLASHINFER_MLA_SPARSE", 8, 64 * 1024),
    ],
)
def test_masked_mha_workspace_guards_long_routing_policy(
    backend_name, tensor_parallel_size, query_len
):
    assert _use_masked_mha(
        backend_name=backend_name,
        tensor_parallel_size=tensor_parallel_size,
        qk_head_dim=256,
        v_head_dim=256,
        query_len=query_len,
        seq_len=query_len,
        has_context=False,
    )
    assert not _masked_mha_workspace_fits(
        batch_size=1,
        max_query_len=query_len,
        max_context_chunk_seq_len=0,
        workspace_numel=GLOBAL_TOPK_MASK_MAX_BYTES // torch.int32.itemsize,
    )


@pytest.mark.parametrize(
    ("model_dims", "kv_cache_dtype", "fa_version", "expected"),
    [
        pytest.param((128, 512, 128, 64, 128), "auto", 4, True, id="deepseek_v32"),
        pytest.param((64, 512, 192, 64, 256), "auto", 4, True, id="glm5"),
        pytest.param((64, 512, 256, 0, 256), "auto", 4, True, id="glm53_flash_nope"),
        pytest.param((64, 512, 256, 0, 256), "fp8", 4, False, id="quantized_kv"),
        pytest.param((64, 512, 256, 0, 256), "auto", 3, False, id="no_fa4"),
        pytest.param((64, 512, 256, 64, 256), "auto", 4, False, id="rope_320"),
        pytest.param((128, 512, 256, 0, 256), "auto", 4, False, id="wrong_heads"),
    ],
)
def test_is_masked_mha_available_model_dims(
    monkeypatch, model_dims, kv_cache_dtype, fa_version, expected
):
    """The allow-list gates masked MHA per exact model geometry: the DeepSeek-V3.2,
    GLM-5 and NoPE GLM-5.3-Flash layouts on an SM100-family GPU with FA4 and an
    unquantized KV cache, nothing else."""
    import vllm.model_executor.layers.attention.sparse_mla_attention as mod

    monkeypatch.setattr(
        mod.current_platform, "is_device_capability_family", lambda family: True
    )
    monkeypatch.setattr(mod, "get_flash_attn_version", lambda **kwargs: fa_version)
    assert _is_masked_mha_available(*model_dims, kv_cache_dtype) is expected


def test_masked_mha_workspace_fits_accounts_for_batch_and_context():
    """Request count and context chunk length are independent multipliers."""
    base = dict(batch_size=2, max_query_len=2048, max_context_chunk_seq_len=2048)
    exact = math.prod(_topk_mask_shape(2, 2048, 2048))

    assert _masked_mha_workspace_fits(**base, workspace_numel=exact)
    assert not _masked_mha_workspace_fits(
        **{**base, "batch_size": 3}, workspace_numel=exact
    )
    assert not _masked_mha_workspace_fits(
        **{**base, "max_context_chunk_seq_len": 4096}, workspace_numel=exact
    )


PREFILL_BATCH_SPECS = {
    "short_dense_mha": BatchSpec(seq_lens=[64, 128], query_lens=[64, 128]),
    "short_context_dense_mha": BatchSpec(seq_lens=[128, 160], query_lens=[64, 32]),
    "masked_mha": BatchSpec(seq_lens=[256], query_lens=[256]),
    "masked_mha_chunked_context": BatchSpec(seq_lens=[448, 384], query_lens=[256, 256]),
}


@pytest.mark.skipif(
    torch.cuda.get_device_capability()[0] < 10,
    reason="Sparse MLA forward_mha requires FA4 (SM100+)",
)
@pytest.mark.parametrize("batch_name", list(PREFILL_BATCH_SPECS.keys()))
@pytest.mark.parametrize("kv_cache_dtype", ["auto"])
@pytest.mark.parametrize(
    ("num_heads", "qk_nope_head_dim", "qk_rope_head_dim", "v_head_dim"),
    [
        pytest.param(128, 128, 64, 128, id="deepseek_hd192_v128"),
        pytest.param(64, 192, 64, 256, id="glm5_hd256_v256"),
        pytest.param(64, 256, 0, 256, id="glm53_flash_nope_hd256_v256"),
    ],
)
def test_sparse_backend_prefill_correctness(
    default_vllm_config,
    dist_init,
    batch_name,
    kv_cache_dtype,
    num_heads,
    qk_nope_head_dim,
    qk_rope_head_dim,
    v_head_dim,
    workspace_init,
):
    """Test dense and masked MHA across supported sparse MLA dimensions."""
    backend_cls = FlashMLASparseBackend
    batch_spec = PREFILL_BATCH_SPECS[batch_name]

    device = torch.device("cuda")
    dtype = torch.bfloat16
    block_size = 64

    kv_lora_rank = 512
    head_size = kv_lora_rank + qk_rope_head_dim
    masked_mha = batch_name.startswith("masked_mha")
    topk_tokens = 200 if masked_mha else 512

    max_seqlen = max(batch_spec.seq_lens)
    total_cache_tokens = sum(batch_spec.seq_lens)

    vllm_config = create_vllm_config(
        model_name="deepseek-ai/DeepSeek-V2-Lite-Chat",
        tensor_parallel_size=1,
        max_model_len=max_seqlen,
        num_gpu_blocks=max(2048, cdiv(total_cache_tokens, block_size) + 1),
        block_size=block_size,
        hf_config_override={
            "index_topk": topk_tokens,
            "attn_module_list_cfg": [{"topk_tokens": topk_tokens}],
        },
    )
    model_config = vllm_config.model_config
    model_config.hf_text_config = SimpleNamespace(
        index_topk=topk_tokens,
        q_lora_rank=None,
        kv_lora_rank=kv_lora_rank,
        qk_nope_head_dim=qk_nope_head_dim,
        qk_rope_head_dim=qk_rope_head_dim,
        v_head_dim=v_head_dim,
        model_type="deepseek_v2",
    )
    del model_config.hf_config.index_topk  # Composite configs only nest this field.
    model_config.dtype = dtype
    model_config.model_arch_config.total_num_attention_heads = num_heads
    model_config.get_num_attention_heads = MethodType(
        lambda self, parallel_config: num_heads, model_config
    )
    model_config.get_num_kv_heads = MethodType(
        lambda self, parallel_config: 1, model_config
    )
    model_config.get_head_size = MethodType(lambda self: head_size, model_config)
    model_config.get_sliding_window = MethodType(lambda self: None, model_config)

    kv_cache_spec = create_standard_kv_cache_spec(vllm_config)
    scale = 1.0 / math.sqrt(qk_nope_head_dim + qk_rope_head_dim)

    torch.manual_seed(42)

    W_UK = torch.rand(
        kv_lora_rank, num_heads, qk_nope_head_dim, dtype=dtype, device=device
    )
    W_UV = torch.rand(kv_lora_rank, num_heads, v_head_dim, dtype=dtype, device=device)

    seq_lens = batch_spec.seq_lens
    query_lens = batch_spec.query_lens

    # Compute dense reference outputs.
    total_query_tokens = sum(query_lens)
    sparse_indices = torch.full(
        (total_query_tokens, topk_tokens), -1, dtype=torch.int32, device=device
    )

    all_q, all_kv_c_new, all_k_pe_new = [], [], []
    kv_c_contexts, k_pe_contexts = [], []
    reference_outputs = []
    global_token_idx = 0

    for i in range(batch_spec.batch_size):
        s_len = seq_lens[i]
        q_len = query_lens[i]
        ctx_len = s_len - q_len

        q_mha = torch.rand(
            q_len,
            num_heads,
            qk_nope_head_dim + qk_rope_head_dim,
            dtype=dtype,
            device=device,
        )
        kv_c_full = torch.rand(s_len, kv_lora_rank, dtype=dtype, device=device)
        k_pe_full = torch.rand(s_len, 1, qk_rope_head_dim, dtype=dtype, device=device)

        # Decompress all KV for reference
        kv_b_weight = torch.cat([W_UK, W_UV], dim=-1).view(
            kv_lora_rank, num_heads * (qk_nope_head_dim + v_head_dim)
        )
        kv_decompressed = (kv_c_full @ kv_b_weight).view(
            s_len, num_heads, qk_nope_head_dim + v_head_dim
        )
        k_nope_all, v_all = kv_decompressed.split(
            [qk_nope_head_dim, v_head_dim], dim=-1
        )
        k_pe_expanded = k_pe_full.expand(-1, num_heads, -1)
        k_all = torch.cat([k_nope_all, k_pe_expanded], dim=-1)

        for j in range(q_len):
            attend_end = ctx_len + j + 1
            q_tok = q_mha[j : j + 1]  # (1, H, D_qk)
            if masked_mha:
                actual_topk = min(topk_tokens, attend_end)
                attend_indices = torch.randperm(attend_end, device=device)[:actual_topk]
                sparse_indices[global_token_idx, :actual_topk] = attend_indices
                k_attend = k_all[attend_indices]
                v_attend = v_all[attend_indices]
            else:
                k_attend = k_all[:attend_end]  # (N, H, D_qk)
                v_attend = v_all[:attend_end]  # (N, H, D_v)

            q_sdpa = q_tok.unsqueeze(0).transpose(1, 2).float()
            k_sdpa = k_attend.unsqueeze(0).transpose(1, 2).float()
            v_sdpa = v_attend.unsqueeze(0).transpose(1, 2).float()

            out = torch.nn.functional.scaled_dot_product_attention(
                q_sdpa, k_sdpa, v_sdpa, scale=scale
            )
            out = out.transpose(1, 2).squeeze(0)  # (1, H, D_v)
            reference_outputs.append(out.to(dtype).flatten(start_dim=-2))
            global_token_idx += 1

        all_q.append(q_mha)
        all_kv_c_new.append(kv_c_full[ctx_len:])
        all_k_pe_new.append(k_pe_full[ctx_len:])
        kv_c_contexts.append(kv_c_full)
        k_pe_contexts.append(k_pe_full)

    query_cat = torch.cat(all_q, dim=0)
    kv_c_cat = torch.cat(all_kv_c_new, dim=0)
    k_pe_cat = torch.cat(all_k_pe_new, dim=0)
    ref_output = torch.cat(reference_outputs, dim=0)

    vllm_config.cache_config.cache_dtype = kv_cache_dtype
    vllm_config.model_config.hf_text_config.index_topk = topk_tokens

    common_attn_metadata = create_common_attn_metadata(
        batch_spec,
        vllm_config.cache_config.block_size,
        device,
        arange_block_indices=True,
    )

    kv_cache = create_and_prepopulate_kv_cache(
        kv_c_contexts=kv_c_contexts,
        k_pe_contexts=k_pe_contexts,
        block_size=block_size,
        head_size=head_size,
        dtype=dtype,
        device=device,
        num_blocks=vllm_config.cache_config.num_gpu_blocks,
        common_attn_metadata=common_attn_metadata,
        randomize_blocks=False,
        kv_cache_dtype=kv_cache_dtype,
    )

    # The sparse builder clones the layer's dense-MHA prefill backend from
    # static_forward_context; register a mock layer carrying one.
    prefill_backend = get_mla_prefill_backend(vllm_config)(
        num_heads=num_heads,
        scale=scale,
        kv_lora_rank=kv_lora_rank,
        qk_nope_head_dim=qk_nope_head_dim,
        qk_rope_head_dim=qk_rope_head_dim,
        v_head_dim=v_head_dim,
        vllm_config=vllm_config,
    )
    vllm_config.compilation_config.static_forward_context["placeholder"] = (
        SimpleNamespace(prefill_backend=prefill_backend)
    )

    builder_cls = backend_cls.get_builder_cls()
    builder = builder_cls(kv_cache_spec, ["placeholder"], vllm_config, device)
    if batch_name == "masked_mha_chunked_context":
        builder.chunked_prefill_workspace_size = block_size * batch_spec.batch_size
        builder.chunked_prefill_workspace = torch.empty(
            (builder.chunked_prefill_workspace_size, head_size),
            dtype=dtype,
            device=device,
        )
    # Drive the queries through the dense-MHA prefill path directly (the routing
    # threshold would otherwise classify these short queries as MQA decodes).
    builder.reorder_batch_threshold = 1
    metadata = builder.build(
        common_prefix_len=0, common_attn_metadata=common_attn_metadata
    )

    mock_indexer = SimpleNamespace(topk_indices_buffer=sparse_indices)

    kv_b_proj_weight = torch.cat([W_UK, W_UV], dim=-1).view(
        kv_lora_rank, num_heads * (qk_nope_head_dim + v_head_dim)
    )

    mock_kv_b_proj = ColumnParallelLinear(
        input_size=kv_lora_rank,
        output_size=num_heads * (qk_nope_head_dim + v_head_dim),
        bias=False,
    ).to(device=device, dtype=dtype)
    mock_kv_b_proj.weight = torch.nn.Parameter(kv_b_proj_weight.T.contiguous())

    impl_cls = backend_cls.get_impl_cls()
    with set_current_vllm_config(vllm_config):
        impl = impl_cls(
            num_heads=num_heads,
            head_size=head_size,
            scale=scale,
            num_kv_heads=1,
            alibi_slopes=None,
            sliding_window=None,
            kv_cache_dtype=kv_cache_dtype,
            logits_soft_cap=None,
            attn_type="decoder",
            kv_sharing_target_layer_name=None,
            q_lora_rank=None,
            kv_lora_rank=kv_lora_rank,
            qk_nope_head_dim=qk_nope_head_dim,
            qk_rope_head_dim=qk_rope_head_dim,
            qk_head_dim=qk_nope_head_dim + qk_rope_head_dim,
            v_head_dim=v_head_dim,
            kv_b_proj=mock_kv_b_proj,
            indexer=mock_indexer,
        )
        impl.process_weights_after_loading(dtype)

    out_buffer = torch.empty(
        total_query_tokens, num_heads * v_head_dim, dtype=dtype, device=device
    )

    with torch.inference_mode():
        impl.forward_mha(
            q=query_cat,
            kv_c_normed=kv_c_cat,
            k_pe=k_pe_cat,
            # Impls see the bind-time-squeezed [B, N, C] cache; mirror bind_kv_cache.
            kv_c_and_k_pe_cache=kv_cache.squeeze(1),
            attn_metadata=metadata,
            k_scale=torch.tensor(1.0, device=device),
            output=out_buffer,
        )

    assert out_buffer.shape == ref_output.shape
    assert torch.isfinite(out_buffer).all(), "Non-finite values in output"
    torch.testing.assert_close(out_buffer, ref_output, rtol=0.01, atol=0.01)


@pytest.mark.parametrize(
    "seq_lens,query_lens,workspace_size,max_logits_bytes,expected",
    [
        (
            torch.tensor([0]),
            torch.tensor([0]),
            100,
            1000,
            [],
        ),
        # Logits constraint triggers split (M*N exceeds budget)
        # req0: M=10, N=100 -> 1000 elems (4000 bytes) - fits in 5000
        # req1: adding M=10, N=100 -> new_M=20, new_N=200 -> 4000 elems > 1250
        (
            torch.tensor([100, 100, 100]),
            torch.tensor([10, 10, 10]),
            1000,  # workspace allows all
            5000,  # 1250 float32 elems -> forces split
            [
                (slice(0, 1), slice(0, 10)),
                (slice(1, 2), slice(0, 10)),
                (slice(2, 3), slice(0, 10)),
            ],
        ),
        # Both constraints satisfied - all fit in one chunk
        (
            torch.tensor([10, 10, 10]),
            torch.tensor([5, 5, 5]),
            100,
            10000,  # 2500 elems, M*N = 15*30 = 450 < 2500
            [(slice(0, 3), slice(0, 15))],
        ),
        # Workspace constraint triggers first
        (
            torch.tensor([50, 50, 50]),
            torch.tensor([1, 1, 1]),
            50,  # workspace only fits one at a time
            1000000,  # logits budget is huge
            [
                (slice(0, 1), slice(0, 1)),
                (slice(1, 2), slice(0, 1)),
                (slice(2, 3), slice(0, 1)),
            ],
        ),
        # Greedy filling: first two fit, third doesn't
        # req0: M=5, N=10 -> 50 elems
        # req0+1: M=10, N=20 -> 200 elems <= 250
        # req0+1+2: M=15, N=30 -> 450 elems > 250
        (
            torch.tensor([10, 10, 10]),
            torch.tensor([5, 5, 5]),
            100,
            1000,  # 250 elems
            [(slice(0, 2), slice(0, 10)), (slice(2, 3), slice(0, 5))],
        ),
    ],
)
def test_split_indexer_prefill_chunks(
    seq_lens, query_lens, workspace_size, max_logits_bytes, expected
):
    out = DeepseekV32IndexerMetadataBuilder._split_indexer_prefill_chunks(
        seq_lens,
        query_lens,
        workspace_size,
        max_logits_bytes,
    )
    assert out == expected


def test_split_indexer_prefill_chunks_single_request_overflow():
    """Test that single request exceeding budget is sub-chunked on query dim."""
    seq_lens = torch.tensor([1000, 50])
    query_lens = torch.tensor([100, 5])

    out = DeepseekV32IndexerMetadataBuilder._split_indexer_prefill_chunks(
        seq_lens, query_lens, 2000, 1000
    )
    # max_logits_elems = 250, N=1000 -> max_q = 1 -> 100 query sub-chunks
    expected = [(slice(0, 1), slice(i, i + 1)) for i in range(100)]
    # req1: M=5, N=50 -> 250 elems fits budget
    expected.append((slice(1, 2), slice(0, 5)))
    assert out == expected


# 384 is not a power of two, so it counts via the tiled atomic accumulation
# rather than the single-tile path 128 takes.
@pytest.mark.parametrize("num_topk_tokens", [128, 384])
def test_triton_convert_returns_valid_counts(num_topk_tokens: int):
    """Test that return_valid_counts correctly counts non-negative indices."""
    device = torch.device(DEVICE_TYPE)
    num_tokens = 8
    num_requests = 2
    max_blocks_per_req = 10
    block_size = 64

    req_id = torch.tensor([0, 0, 0, 0, 1, 1, 1, 1], dtype=torch.int32, device=device)
    block_table = torch.arange(
        num_requests * max_blocks_per_req, dtype=torch.int32, device=device
    ).view(num_requests, max_blocks_per_req)

    # Create token indices with varying numbers of valid entries: half the row,
    # a quarter of it, the whole row, then a single valid entry -- twice over.
    token_indices = torch.full(
        (num_tokens, num_topk_tokens), -1, dtype=torch.int32, device=device
    )
    valid_counts_per_token = [
        num_topk_tokens // 2,
        num_topk_tokens // 4,
        num_topk_tokens,
        1,
    ] * 2
    expected_valid = []
    for i in range(num_tokens):
        num_valid = valid_counts_per_token[i]
        token_indices[i, :num_valid] = torch.arange(
            num_valid, dtype=torch.int32, device=device
        ) % (block_size * max_blocks_per_req)
        expected_valid.append(num_valid)

    expected_valid_tensor = torch.tensor(
        expected_valid, dtype=torch.int32, device=device
    )

    # Test with return_valid_counts=True
    result, valid_counts = triton_convert_req_index_to_global_index(
        req_id,
        block_table,
        token_indices,
        BLOCK_SIZE=block_size,
        NUM_TOPK_TOKENS=num_topk_tokens,
        return_valid_counts=True,
    )

    torch.testing.assert_close(valid_counts, expected_valid_tensor, rtol=0, atol=0)

    # Test that return_valid_counts=False returns only the indices
    result_only = triton_convert_req_index_to_global_index(
        req_id,
        block_table,
        token_indices,
        BLOCK_SIZE=block_size,
        NUM_TOPK_TOKENS=num_topk_tokens,
        return_valid_counts=False,
    )
    assert isinstance(result_only, torch.Tensor)
    for row, num_valid in enumerate(expected_valid):
        compact_valid = result[row, :num_valid].sort().values
        original_valid = result_only[row][result_only[row] >= 0].sort().values
        torch.testing.assert_close(compact_valid, original_valid, rtol=0, atol=0)
        assert torch.all(result[row, num_valid:] == -1)


def test_sparse_mla_index_group_converts_decode_indices_once(monkeypatch):
    device = torch.device(DEVICE_TYPE)
    logical = torch.full((2, 128), -1, dtype=torch.int32, device=device)
    logical[:, :4] = torch.tensor(
        [[0, 2, 5, -1], [1, 3, 7, -1]], dtype=torch.int32, device=device
    )
    builder = SparseMLAIndexGroupBuilder(logical)
    group, leader_index = builder.register_layer(True)
    follower_group, follower_index = builder.register_layer(False)
    assert follower_group is group

    block_table = torch.tensor([[2, 3], [4, 5]], dtype=torch.int32, device=device)
    metadata = SimpleNamespace(
        block_table=block_table,
        block_size=4,
        req_id_per_token=torch.tensor([0, 1], dtype=torch.int32, device=device),
        num_decode_tokens=2,
        num_actual_tokens=2,
    )
    calls = []
    convert = index_group_module.triton_convert_req_index_to_global_index

    def convert_spy(*args, **kwargs):
        calls.append((args, kwargs))
        return convert(*args, **kwargs)

    monkeypatch.setattr(
        index_group_module,
        "triton_convert_req_index_to_global_index",
        convert_spy,
    )
    group.set_logical_topk_ready(leader_index)
    leader_result = group.convert_logical_to_physical_topk(
        leader_index,
        logical,
        metadata,
        block_stride_rows=None,
        return_valid_counts=True,
    )
    follower_result = group.convert_logical_to_physical_topk(
        follower_index,
        logical,
        metadata,
        block_stride_rows=None,
        return_valid_counts=True,
    )
    torch.accelerator.synchronize()

    assert len(calls) == 1
    leader_indices, leader_counts = leader_result
    follower_indices, follower_counts = follower_result
    torch.testing.assert_close(follower_indices, leader_indices)
    torch.testing.assert_close(follower_counts, leader_counts)


def test_sparse_mla_index_group_falls_back_for_prefill_sized_batch(monkeypatch):
    device = torch.device(DEVICE_TYPE)
    logical = torch.full((4, 128), -1, dtype=torch.int32, device=device)
    logical[:, 0] = torch.tensor([0, 1, 2, 3], dtype=torch.int32, device=device)
    builder = SparseMLAIndexGroupBuilder(logical, max_decode_rows=2)
    group, leader_index = builder.register_layer(True)
    follower_group, follower_index = builder.register_layer(False)
    assert follower_group is group

    metadata = SimpleNamespace(
        block_table=torch.tensor(
            [[2], [3], [4], [5]], dtype=torch.int32, device=device
        ),
        block_size=4,
        req_id_per_token=torch.arange(4, dtype=torch.int32, device=device),
    )
    calls = 0
    convert = index_group_module.triton_convert_req_index_to_global_index

    def convert_spy(*args, **kwargs):
        nonlocal calls
        calls += 1
        return convert(*args, **kwargs)

    monkeypatch.setattr(
        index_group_module,
        "triton_convert_req_index_to_global_index",
        convert_spy,
    )
    leader_result = group.convert_logical_to_physical_topk(
        leader_index,
        logical,
        metadata,
        block_stride_rows=None,
        return_valid_counts=False,
    )
    follower_result = group.convert_logical_to_physical_topk(
        follower_index,
        logical,
        metadata,
        block_stride_rows=None,
        return_valid_counts=False,
    )
    torch.accelerator.synchronize()

    assert calls == 2
    torch.testing.assert_close(follower_result, leader_result)


def test_sparse_mla_index_groups_own_distinct_physical_buffers():
    logical = torch.empty((8, 128), dtype=torch.int32, device=DEVICE_TYPE)
    builder = SparseMLAIndexGroupBuilder(logical, max_decode_rows=2)

    first, _ = builder.register_layer(True)
    follower, _ = builder.register_layer(False)
    second, _ = builder.register_layer(True)

    assert follower is first
    assert second is not first
    assert first.physical_topk_indices[:2].shape == (2, 128)
    assert second.physical_topk_indices[:2].shape == (2, 128)
    assert first.physical_topk_indices.data_ptr() != (
        second.physical_topk_indices.data_ptr()
    )


# HiSparse is host-resident-only and kernel-only: runtime construction
# raises without the compiled CUDA ops.
requires_hisparse_ops = pytest.mark.skipif(
    not _has_hisparse_ops(),
    reason="HiSparse CUDA ops not compiled",
)


def _make_hisparse_runtime(
    *,
    top_k: int = 4,
    device_buffer_size: int = 5,
    max_num_reqs: int = 2,
    row_width: int = 8,
    block_size: int = 64,
    max_swap_rows: int | None = None,
    index_group: hisparse_runtime.HiSparseIndexGroup | None = None,
) -> HiSparseRuntime:
    runtime = HiSparseRuntime(
        config=ResolvedHiSparseConfig(
            top_k=top_k,
            device_buffer_size=device_buffer_size,
        ),
        max_num_reqs=max_num_reqs,
        row_width=row_width,
        kv_dtype=torch.float32,
        device=DEVICE_TYPE,
        max_swap_rows=max_swap_rows,
        index_group=index_group,
    )
    blocks_per_request = cdiv(runtime.region_stride, block_size)
    # Leave one extra physical block for resident-tier tests. HMA group layouts
    # alias this backing, but distinct group allocations use distinct block IDs.
    num_blocks = max_num_reqs * blocks_per_request + 1
    raw = torch.zeros(
        num_blocks * block_size * row_width,
        dtype=torch.float32,
        device=DEVICE_TYPE,
    ).view(torch.int8)
    block_table = torch.arange(
        max_num_reqs * blocks_per_request,
        dtype=torch.int32,
        device=DEVICE_TYPE,
    ).view(max_num_reqs, blocks_per_request)
    runtime.bind_hot_cache(
        raw,
        byte_offset=0,
        block_stride=block_size * row_width * torch.float32.itemsize,
        num_blocks=num_blocks,
        block_size=block_size,
        block_table=block_table,
    )
    runtime.request_state_indices = torch.arange(
        max_num_reqs, dtype=torch.int32, device=runtime.device
    )
    return runtime


def _make_hisparse_cache_handle(
    *,
    top_k: int = 4,
    device_buffer_size: int = 5,
    max_num_reqs: int = 2,
    row_width: int = 8,
    block_size: int = 64,
    max_swap_rows: int | None = None,
) -> HiSparseCacheHandle:
    runtime = _make_hisparse_runtime(
        top_k=top_k,
        device_buffer_size=device_buffer_size,
        max_num_reqs=max_num_reqs,
        row_width=row_width,
        block_size=block_size,
        max_swap_rows=max_swap_rows,
    )
    return HiSparseCacheHandle(runtime)


@requires_hisparse_ops
def test_hisparse_uses_graph_stable_request_state_mapping():
    device = torch.device(DEVICE_TYPE)
    block_size, row_width = 64, 8
    runtime = _make_hisparse_runtime(
        top_k=1,
        device_buffer_size=2,
        max_num_reqs=2,
        row_width=row_width,
        block_size=block_size,
    )
    runtime.bind_source_cache(
        torch.arange(2 * block_size * row_width, dtype=torch.float32)
        .view(2, block_size, row_width)
        .pin_memory()
    )
    runtime.request_state_indices = torch.tensor([1], dtype=torch.int32, device=device)

    cache_handle = HiSparseCacheHandle(runtime)
    cache_handle.all_context_pages_resident = False
    runtime.begin_forward()
    cache_handle.swap_in(
        req_id_per_token=torch.tensor([0], dtype=torch.int32, device=device),
        block_table=torch.tensor([[1]], dtype=torch.int32, device=device),
        logical_topk_indices=torch.tensor([[0]], dtype=torch.int32, device=device),
        block_size=block_size,
    )
    torch.accelerator.synchronize()

    assert (runtime.index_group.device_global_indices[0] == -1).all()
    assert (runtime.index_group.device_global_indices[1] == block_size).any()


@requires_hisparse_ops
def test_hisparse_maps_speculative_rows_through_request_state():
    """Multiple verification rows for one request share its persistent state."""
    device = torch.device(DEVICE_TYPE)
    block_size, row_width = 64, 8
    runtime = _make_hisparse_runtime(
        top_k=1,
        device_buffer_size=2,
        max_num_reqs=1,
        row_width=row_width,
        block_size=block_size,
        max_swap_rows=4,
    )
    runtime.bind_source_cache(
        torch.zeros(1, block_size, row_width, dtype=torch.float32).pin_memory()
    )

    cache_handle = HiSparseCacheHandle(runtime)
    resident_table = torch.tensor([[1]], dtype=torch.int32, device=device)
    cache_handle.bind_cache(
        runtime.hot.cache.view(torch.int8),
        byte_offset=0,
        block_stride=block_size * row_width * torch.float32.itemsize,
        num_blocks=2,
        block_size=block_size,
        block_table=resident_table,
        slot_mapping=torch.arange(4, dtype=torch.int64, device=device),
    )
    cache_handle.all_context_pages_resident = True
    runtime.begin_forward()
    physical = cache_handle.swap_in(
        req_id_per_token=torch.zeros(4, dtype=torch.int32, device=device),
        block_table=torch.zeros((1, 1), dtype=torch.int32, device=device),
        logical_topk_indices=torch.zeros((4, 1), dtype=torch.int32, device=device),
        block_size=block_size,
    )
    torch.accelerator.synchronize()

    assert physical.tolist() == [[block_size]] * 4


@requires_hisparse_ops
def test_hisparse_resident_rows_bypass_hot_lru():
    device = torch.device(DEVICE_TYPE)
    block_size, row_width = 64, 8
    cache_handle = _make_hisparse_cache_handle(
        top_k=4,
        device_buffer_size=5,
        max_num_reqs=1,
        row_width=row_width,
        block_size=block_size,
    )
    raw = cache_handle.runtime.hot.cache.view(torch.int8)
    resident_table = torch.tensor([[1]], dtype=torch.int32, device=device)
    resident_slots = torch.tensor([block_size], dtype=torch.int64, device=device)
    cache_handle.bind_cache(
        raw,
        byte_offset=0,
        block_stride=block_size * row_width * torch.float32.itemsize,
        num_blocks=2,
        block_size=block_size,
        block_table=resident_table,
        slot_mapping=resident_slots,
    )

    host = torch.randn(1, block_size, row_width).pin_memory()
    cache_handle.runtime.bind_source_cache(host)
    resident_rows = host[0].to(device).add(7)
    cache_handle.runtime.hot.cache[1].copy_(resident_rows)
    source_table = torch.tensor([[5]], dtype=torch.int32, device=device)
    request_ids = torch.zeros(1, dtype=torch.int32, device=device)
    topk = torch.tensor([[1, 2, 3, 4]], dtype=torch.int32, device=device)
    lru_before = cache_handle.runtime.index_group.lru_slots.clone()

    cache_handle.runtime.begin_forward()
    indices, valid_counts = cache_handle.swap_in(
        req_id_per_token=request_ids,
        block_table=source_table,
        logical_topk_indices=topk,
        block_size=block_size,
        return_valid_counts=True,
    )
    torch.accelerator.synchronize()

    expected_indices = topk + block_size
    torch.testing.assert_close(indices, expected_indices)
    torch.testing.assert_close(
        valid_counts, torch.tensor([topk.shape[1]], dtype=torch.int32, device=device)
    )
    torch.testing.assert_close(cache_handle.runtime.index_group.lru_slots, lru_before)
    torch.testing.assert_close(
        cache_handle.runtime.hot.attention_cache.view(-1, row_width)[
            indices.to(torch.long)
        ],
        resident_rows[topk.to(torch.long)],
    )


@requires_hisparse_ops
def test_hisparse_swap_in_preserves_rows_across_eviction():
    device = torch.device(DEVICE_TYPE)
    block_size = 64
    row_width = 64
    num_blocks = 64
    top_k = 128
    buf = 256
    num_reqs = 4

    kv_pool = torch.randn(
        (num_blocks + 1, block_size, row_width), dtype=torch.float32
    ).pin_memory()
    flat_pool = kv_pool.reshape(-1, row_width)

    runtime = _make_hisparse_runtime(
        top_k=top_k,
        device_buffer_size=buf,
        max_num_reqs=num_reqs,
        row_width=row_width,
    )
    runtime.bind_source_cache(kv_pool)
    cache = HiSparseCacheHandle(runtime)
    cache.all_context_pages_resident = False
    cache.runtime.begin_forward()

    blocks_per_req = num_blocks // num_reqs
    block_table = torch.arange(
        1, num_blocks + 1, dtype=torch.int32, device=device
    ).view(num_reqs, blocks_per_req)
    req_ids = torch.arange(num_reqs, dtype=torch.int32, device=device)
    seq_len = blocks_per_req * block_size
    base = torch.arange(top_k, dtype=torch.int32, device=device)
    for step in range(3):
        cache.runtime.begin_forward()
        topk = torch.stack(
            [(base + step * top_k + row * 17) % seq_len for row in range(num_reqs)]
        )
        topk[:, -1] = -1

        hot_indices, valid_counts = cache.swap_in(
            req_id_per_token=req_ids,
            block_table=block_table,
            logical_topk_indices=topk.clone(),
            block_size=block_size,
            return_valid_counts=True,
        )
        torch.accelerator.synchronize()

        global_ref = _triton_convert_reference_impl(
            req_ids, block_table, topk, block_size, top_k
        )
        valid = hot_indices >= 0
        torch.testing.assert_close(
            valid_counts,
            (global_ref >= 0).sum(dim=1, dtype=torch.int32),
            rtol=0,
            atol=0,
        )
        gathered = cache.runtime.hot.attention_cache.reshape(-1, row_width)[
            hot_indices[valid].to(torch.long)
        ].cpu()
        expected = flat_pool[global_ref[valid].cpu().to(torch.long)]
        torch.testing.assert_close(gathered, expected)


@requires_hisparse_ops
def test_hisparse_multi_step_swaps_match_independent():
    """The shared LRU must retain every speculative step until followers run."""
    device = torch.device(DEVICE_TYPE)
    torch.manual_seed(0)
    block_size = 64
    row_width = 64
    num_blocks = 64
    top_k = 128
    buf = 3 * top_k
    num_reqs = 4

    kv_pool = torch.randn(
        (num_blocks, block_size, row_width), dtype=torch.float32
    ).pin_memory()

    def make(
        index_group: hisparse_runtime.HiSparseIndexGroup | None = None,
    ) -> HiSparseCacheHandle:
        runtime = _make_hisparse_runtime(
            top_k=top_k,
            max_num_reqs=num_reqs,
            device_buffer_size=buf,
            row_width=row_width,
            block_size=block_size,
            max_swap_rows=2 * num_reqs,
            index_group=index_group,
        )
        runtime.bind_source_cache(kv_pool)
        cache = HiSparseCacheHandle(runtime)
        cache.all_context_pages_resident = False
        return cache

    blocks_per_req = num_blocks // num_reqs
    block_table = torch.arange(num_blocks, dtype=torch.int32, device=device).view(
        num_reqs, blocks_per_req
    )
    req_ids = torch.arange(num_reqs, dtype=torch.int32, device=device)
    seq_len = blocks_per_req * block_size
    producer = make()
    shared = make(producer.runtime.index_group)
    indep = make()
    base = torch.arange(top_k, dtype=torch.int32, device=device)
    for iteration in range(2):
        producer.runtime.begin_forward()
        shared.runtime.begin_forward()
        indep.runtime.begin_forward()
        topks = [
            torch.stack(
                [
                    (base + (iteration * 2 + step) * top_k + row * 17) % seq_len
                    for row in range(num_reqs)
                ]
            )
            for step in range(2)
        ]
        kw = dict(
            req_id_per_token=req_ids,
            block_table=block_table,
            block_size=block_size,
        )
        producer_indices = [
            producer.swap_in(logical_topk_indices=topk.clone(), **kw) for topk in topks
        ]
        independent_indices = [
            indep.swap_in(logical_topk_indices=topk.clone(), **kw) for topk in topks
        ]
        follower_indices = [
            shared.swap_in(logical_topk_indices=topk.clone(), **kw) for topk in topks
        ]
        torch.accelerator.synchronize()

        for idx_full, idx_indep, idx_shared in zip(
            producer_indices, independent_indices, follower_indices
        ):
            torch.testing.assert_close(idx_shared, idx_full, rtol=0, atol=0)
            torch.testing.assert_close(idx_indep, idx_full, rtol=0, atol=0)
        torch.testing.assert_close(shared.runtime.hot.cache, indep.runtime.hot.cache)

        shared_rows = shared.runtime.hot.attention_cache.reshape(-1, row_width)
        host_rows = kv_pool.reshape(-1, row_width)
        for physical_rows, logical_rows in zip(follower_indices, topks):
            global_rows = _triton_convert_reference_impl(
                req_ids, block_table, logical_rows, block_size, top_k
            )
            valid = physical_rows >= 0
            torch.testing.assert_close(
                shared_rows[physical_rows[valid].to(torch.long)].cpu(),
                host_rows[global_rows[valid].cpu().to(torch.long)],
            )


@requires_hisparse_ops
def test_hisparse_multi_step_writes_request_major_output():
    device = torch.device(DEVICE_TYPE)
    block_size = 64
    row_width = 8
    num_reqs = 2
    query_len = 3
    top_k = 4
    cache = _make_hisparse_cache_handle(
        top_k=top_k,
        device_buffer_size=query_len * top_k,
        max_num_reqs=num_reqs,
        row_width=row_width,
        block_size=block_size,
        max_swap_rows=num_reqs * query_len,
    )
    host = torch.arange(
        (num_reqs + 1) * block_size * row_width, dtype=torch.float32
    ).view(num_reqs + 1, block_size, row_width)
    cache.runtime.bind_source_cache(host.pin_memory())
    block_table = torch.arange(1, num_reqs + 1, dtype=torch.int32, device=device).view(
        -1, 1
    )
    request_ids = torch.arange(num_reqs, dtype=torch.int32, device=device)
    logical = torch.tensor(
        [
            [[0, 1, 2, 3], [4, 5, 6, 7], [8, 9, 10, 11]],
            [[12, 13, 14, 15], [16, 17, 18, 19], [20, 21, 22, 23]],
        ],
        dtype=torch.int32,
        device=device,
    )
    physical = torch.full_like(logical, -1)
    valid_counts = torch.zeros((num_reqs, query_len), dtype=torch.int32, device=device)

    cache.runtime.begin_forward()
    for step in range(query_len):
        cache.swap_in(
            request_ids,
            block_table,
            logical[:, step],
            block_size=block_size,
            return_valid_counts=True,
            attention_indices_out=physical[:, step],
            valid_counts_out=valid_counts[:, step],
        )
    torch.accelerator.synchronize()

    torch.testing.assert_close(
        valid_counts,
        torch.full_like(valid_counts, top_k),
    )
    global_rows = _triton_convert_reference_impl(
        torch.arange(num_reqs, device=device, dtype=torch.int32).repeat_interleave(
            query_len
        ),
        block_table,
        logical.view(num_reqs * query_len, top_k),
        block_size,
        top_k,
    )
    hot_rows = cache.runtime.hot.attention_cache.reshape(-1, row_width)
    host_rows = host.reshape(-1, row_width)
    torch.testing.assert_close(
        hot_rows[physical.view(num_reqs * query_len, top_k).to(torch.long)].cpu(),
        host_rows[global_rows.cpu().to(torch.long)],
    )


@requires_hisparse_ops
def test_hisparse_kv_update_writes_resident_and_staging_caches():
    device = torch.device(DEVICE_TYPE)
    block_size = 4
    row_width = 8
    cache_handle = _make_hisparse_cache_handle(
        max_num_reqs=1,
        row_width=row_width,
        block_size=block_size,
    )
    resident_slots = torch.tensor([4, 5, -1], dtype=torch.int64, device=device)
    cache_handle.bind_cache(
        cache_handle.runtime.hot.cache.view(torch.int8),
        byte_offset=0,
        block_stride=block_size * row_width * torch.float32.itemsize,
        num_blocks=cache_handle.runtime.hot.cache.shape[0],
        block_size=block_size,
        block_table=torch.tensor([[1]], dtype=torch.int32, device=device),
        slot_mapping=resident_slots,
    )
    cache_handle.mirror_staging_cache = torch.empty(
        (1, block_size, row_width), dtype=torch.float32, device=device
    )
    cache_handle.mirror_staging_slots = torch.arange(
        block_size, dtype=torch.int64, device=device
    )
    slots = torch.tensor([3, 7, -1], dtype=torch.int64, device=device)
    kv_c = torch.randn(8, row_width - 2, device=device)
    k_pe = torch.randn(8, 1, 2, device=device)
    cache_handle.num_actual_tokens = slots.numel()
    cache_handle.decode_batch = False
    cache_handle.host_mirror_required = True
    source_cache = torch.zeros_like(cache_handle.view.cache)
    impl = object.__new__(FlashMLASparseImpl)
    layer = SimpleNamespace(
        hisparse_cache=cache_handle,
        use_pcp=False,
        impl=impl,
    )
    mla_attention.MLAAttention.update_kv_cache(
        layer,
        kv_c,
        k_pe,
        source_cache,
        slots,
        SimpleNamespace(num_decode_tokens=0),
        "auto",
        torch.tensor(1.0, device=device),
    )
    torch.accelerator.synchronize()

    expected = torch.cat([kv_c[:2], k_pe[:2, 0]], dim=-1).cpu()
    torch.testing.assert_close(
        cache_handle.view.cache.view(-1, row_width)[resident_slots[:2]],
        expected.to(device),
    )
    torch.testing.assert_close(source_cache, torch.zeros_like(source_cache))
    staged = cache_handle.mirror_staging_cache.view(-1, row_width)
    staged_expected = torch.cat([kv_c[:3], k_pe[:3, 0]], dim=-1)
    torch.testing.assert_close(staged[:3], staged_expected)


@requires_hisparse_ops
def test_hisparse_row_dma_copies_discontiguous_spans_across_layers():
    device = torch.device(DEVICE_TYPE)
    block_size = 4
    row_width = 8
    num_rows = 16
    resident_caches = tuple(
        (
            torch.arange(num_rows * row_width, dtype=torch.uint8, device=device)
            .add_(layer_index * 17)
            .view(num_rows // block_size, block_size, row_width)
        )
        for layer_index in range(2)
    )
    host_caches = tuple(
        torch.zeros((num_rows, row_width), dtype=torch.uint8, pin_memory=True)
        for _ in resident_caches
    )
    worker = object.__new__(HiSparseConnectorWorker)
    worker.is_host_writer = True
    worker.kernel_block_size = block_size
    worker.resident_caches = resident_caches
    worker.host_caches = host_caches
    worker.cache_handles = [
        SimpleNamespace(
            decode_batch=True,
            runtime=SimpleNamespace(resident_source_index=0),
        )
        for _ in resident_caches
    ]
    worker.dma_stream = torch.cuda.Stream(device=device)
    worker.host_write_event = torch.Event()
    worker._dma_free_descriptors = []
    worker._pending_dma_descriptors = deque()
    worker._pending_transfer_events = deque()
    worker._enqueued_transfer_ids = []
    worker._dma_submitted = False
    worker._set_row_mirrors(
        (
            SparseKVRowMirror((1,), 5, 2),
            SparseKVRowMirror((8,), 10, 4),
        )
    )

    worker._enqueue_row_dma(range(2))
    current_stream().wait_event(worker.host_write_event)
    torch.accelerator.synchronize()

    for resident, host in zip(resident_caches, host_caches):
        flat_resident = resident.view(num_rows, row_width).cpu()
        torch.testing.assert_close(host[5:7], flat_resident[1:3])
        torch.testing.assert_close(host[10:14], flat_resident[8:12])


@requires_hisparse_ops
def test_hisparse_remaps_strided_hma_rows_for_attention():
    device = torch.device(DEVICE_TYPE)
    block_size, row_width = 4, 8
    runtime = _make_hisparse_runtime(
        max_num_reqs=1,
        block_size=block_size,
        row_width=row_width,
    )
    page_elements = block_size * row_width
    stride_elements = 3 * page_elements
    byte_offset = page_elements * torch.float32.itemsize
    raw = torch.zeros(2 * stride_elements, dtype=torch.float32, device=device).view(
        torch.int8
    )
    runtime.bind_hot_cache(
        raw,
        byte_offset=byte_offset,
        block_stride=stride_elements * torch.float32.itemsize,
        num_blocks=2,
        block_size=block_size,
        block_table=torch.tensor([[0, 1]], dtype=torch.int32, device=device),
    )

    source = (
        torch.arange(36 * row_width, dtype=torch.float32)
        .view(9, block_size, row_width)
        .pin_memory()
    )
    runtime.bind_source_cache(source)
    cache = HiSparseCacheHandle(runtime)
    cache.all_context_pages_resident = False
    runtime.begin_forward()
    attention_indices = cache.swap_in(
        req_id_per_token=torch.tensor([0], dtype=torch.int32, device=device),
        block_table=torch.arange(1, 9, dtype=torch.int32, device=device).view(1, 8),
        logical_topk_indices=torch.tensor(
            [[0, 1, 2, 3]], dtype=torch.int32, device=device
        ),
        block_size=block_size,
    )

    assert runtime.hot.attention_cache is not None
    assert runtime.hot.attention_cache.is_contiguous()
    physical_indices = runtime.index_group.shared_topk.device_topk_rows[0]
    expected = (
        physical_indices // block_size * (stride_elements // row_width)
        + physical_indices % block_size
    )
    torch.testing.assert_close(attention_indices[0], expected)


@requires_hisparse_ops
def test_hisparse_gather_zeroes_unaligned_destination():
    """Invalid host rows are safely zeroed even at a byte-unaligned address."""
    device = torch.device(DEVICE_TYPE)
    row_bytes = 16
    host_cache = torch.ones(1, row_bytes, dtype=torch.uint8).pin_memory()
    storage = torch.full((2 * row_bytes + 1,), 7, dtype=torch.uint8, device=device)
    hot_cache = torch.as_strided(
        storage,
        size=(2, 1, row_bytes),
        stride=(row_bytes + 1, row_bytes, 1),
    )
    global_indices = torch.tensor([[1]], dtype=torch.int32, device=device)
    hot_indices = torch.tensor([[1]], dtype=torch.int32, device=device)
    miss_mask = torch.ones_like(global_indices)

    torch.ops._C_cache_ops.hisparse_gather_plan(
        host_cache,
        hot_cache,
        global_indices,
        hot_indices,
        miss_mask,
        None,
    )
    torch.accelerator.synchronize()

    torch.testing.assert_close(hot_cache[1, 0], torch.zeros_like(hot_cache[1, 0]))
    assert storage[row_bytes].item() == 7


@requires_hisparse_ops
def test_hisparse_gather_rejects_short_attention_block_stride():
    """Reject a stride that would let the gather kernel cross block bounds."""
    device = torch.device(DEVICE_TYPE)
    host_cache = torch.ones(1, 16, dtype=torch.uint8).pin_memory()
    hot_cache = torch.empty((1, 2, 16), dtype=torch.uint8, device=device)
    global_indices = torch.zeros((1, 1), dtype=torch.int32, device=device)
    hot_indices = torch.zeros_like(global_indices)
    miss_mask = torch.ones_like(global_indices)
    attention_indices = torch.empty_like(global_indices)

    with pytest.raises(RuntimeError, match="stride must cover one hot block"):
        torch.ops._C_cache_ops.hisparse_gather_plan(
            host_cache,
            hot_cache,
            global_indices,
            hot_indices,
            miss_mask,
            None,
            attention_indices,
            1,
        )


@requires_hisparse_ops
def test_hisparse_newest_write_and_recycled_slot_invalidation():
    """Newest writes clamp padding and invalidated rows are loaded again."""
    device = torch.device(DEVICE_TYPE)
    block_size = 4
    row_width = 8
    num_blocks = 8

    kv_pool = (
        torch.arange(num_blocks * block_size * row_width, dtype=torch.float32)
        .view(num_blocks, block_size, row_width)
        .pin_memory()
    )
    flat_pool = kv_pool.reshape(-1, row_width)

    cache_handle = _make_hisparse_cache_handle(block_size=block_size)
    cache_handle.runtime.eager_host_mirror = True

    block_table = torch.tensor([[2, 0, 4]], dtype=torch.int32, device=device)
    resident_slots = torch.tensor([4, -1], dtype=torch.int64, device=device)
    cache_handle.bind_cache(
        cache_handle.runtime.hot.cache.view(torch.int8),
        byte_offset=0,
        block_stride=block_size * row_width * torch.float32.itemsize,
        num_blocks=cache_handle.runtime.hot.cache.shape[0],
        block_size=block_size,
        block_table=torch.tensor([[0, 0, 1]], dtype=torch.int32, device=device),
        slot_mapping=resident_slots,
    )
    req_ids = torch.tensor([0], dtype=torch.int32, device=device)
    newest_global = 4 * block_size
    padded_global = 1
    padded_row = flat_pool[padded_global].clone()
    slot_mapping = torch.tensor(
        [newest_global, padded_global], dtype=torch.int64, device=device
    )
    request_state_indices = torch.tensor([0, -1], dtype=torch.int32, device=device)
    cache_handle.runtime.request_state_indices = request_state_indices

    kv_c = torch.randn(3, row_width - 2, device=device)
    k_pe = torch.randn(3, 1, 2, device=device)
    cache_handle.runtime.bind_source_cache(kv_pool)
    cache_handle.num_actual_tokens = slot_mapping.numel()
    ops.concat_and_cache_mla(
        kv_c[:1],
        k_pe[:1, 0],
        cache_handle.view.cache,
        resident_slots[:1],
        kv_cache_dtype="auto",
        scale=torch.tensor(1.0, device=device),
    )
    torch.accelerator.synchronize()
    expected_row = torch.cat([kv_c[0], k_pe[0, 0]]).cpu()
    flat_pool[newest_global].copy_(expected_row)
    torch.testing.assert_close(flat_pool[newest_global], expected_row)
    torch.testing.assert_close(flat_pool[padded_global], padded_row)
    flat_hot = cache_handle.runtime.hot.cache.view(-1, row_width)
    torch.testing.assert_close(flat_hot[resident_slots[0]].cpu(), expected_row)

    topk = torch.tensor([[0, -1, -1, -1]], dtype=torch.int32, device=device)
    cache_handle.all_context_pages_resident = False
    cache_handle.runtime.begin_forward()
    hot_indices = cache_handle.swap_in(
        req_id_per_token=req_ids,
        block_table=block_table,
        logical_topk_indices=topk,
        block_size=block_size,
    )
    torch.accelerator.synchronize()
    stale_hot_slot = hot_indices[0, 0].item()
    stale_row = flat_hot[stale_hot_slot].clone()

    flat_pool[8] += 1000
    cache_handle.runtime.invalidate_written_slots(
        torch.tensor([8], dtype=torch.int64, device=device), req_ids
    )
    cache_handle.runtime.begin_forward()
    hot_indices = cache_handle.swap_in(
        req_id_per_token=req_ids,
        block_table=block_table,
        logical_topk_indices=topk,
        block_size=block_size,
    )
    torch.accelerator.synchronize()
    idx = hot_indices.cpu().tolist()[0][0]
    assert not torch.equal(flat_hot[idx], stale_row)
    torch.testing.assert_close(flat_hot[idx].cpu(), flat_pool[8])


@requires_hisparse_ops
def test_hisparse_mixed_batch_bf16_row_split(
    default_vllm_config, dist_init, workspace_init
):
    """Host-resident mixed batch on the bf16 path is row-split.

    Two long-context speculative-decode requests + one short local-prefill
    chunk: every decode step must be served from the bounded hot buffer before
    it is reused, while only the prefill rows' blocks are staged host->GPU.
    """
    ok, reason = flashmla.is_flashmla_sparse_supported()
    if not ok:
        pytest.skip(reason)

    device = torch.device(DEVICE_TYPE)
    dtype = torch.bfloat16
    torch.manual_seed(0)

    num_heads = 64
    kv_lora_rank = 512
    qk_nope_head_dim = 128
    qk_rope_head_dim = 64
    v_head_dim = 128
    head_size = kv_lora_rank + qk_rope_head_dim
    topk_tokens = 128
    block_size = 64

    # Long decode contexts + a short prefill chunk (router shortcut shape).
    batch_spec = BatchSpec(seq_lens=[2048, 2048, 192], query_lens=[2, 2, 64])
    max_seqlen = max(batch_spec.seq_lens)
    total_cache_tokens = sum(batch_spec.seq_lens)
    total_tokens = batch_spec.compute_num_tokens()

    vllm_config = create_vllm_config(
        model_name="deepseek-ai/DeepSeek-V2-Lite-Chat",
        tensor_parallel_size=1,
        max_model_len=max_seqlen,
        num_gpu_blocks=max(2048, cdiv(total_cache_tokens, block_size) + 1),
        block_size=block_size,
        hf_config_override={
            "index_topk": topk_tokens,
            "attn_module_list_cfg": [{"topk_tokens": topk_tokens}],
        },
    )
    vllm_config.attention_config.hisparse_config = HiSparseConfig(
        device_buffer_size=2 * topk_tokens,
    )
    vllm_config.speculative_config = SpeculativeConfig(
        method="ngram", num_speculative_tokens=1
    )
    model_config = vllm_config.model_config
    model_config.hf_text_config = SimpleNamespace(
        index_topk=topk_tokens,
        q_lora_rank=None,
        kv_lora_rank=kv_lora_rank,
        qk_nope_head_dim=qk_nope_head_dim,
        qk_rope_head_dim=qk_rope_head_dim,
        v_head_dim=v_head_dim,
        model_type="deepseek_v2",
    )
    model_config.dtype = dtype
    model_config.get_num_attention_heads = MethodType(
        lambda self, parallel_config: num_heads, model_config
    )
    model_config.get_num_kv_heads = MethodType(
        lambda self, parallel_config: 1, model_config
    )
    model_config.get_head_size = MethodType(lambda self: head_size, model_config)
    model_config.get_sliding_window = MethodType(lambda self: None, model_config)
    # create_hisparse_cache_handle sizes hot buffers per layer.
    model_config.get_num_layers = MethodType(
        lambda self, parallel_config: 1, model_config
    )

    kv_cache_spec = create_standard_kv_cache_spec(vllm_config)
    common_attn_metadata = create_common_attn_metadata(
        batch_spec,
        vllm_config.cache_config.block_size,
        device,
        arange_block_indices=True,
    )
    # Prepopulate every position of every sequence so the forward needs no
    # KV-cache update (the row split itself is what is under test).
    kv_c_contexts = [
        torch.rand(s_len, kv_lora_rank, dtype=dtype, device=device)
        for s_len in batch_spec.seq_lens
    ]
    k_pe_contexts = [
        torch.rand(s_len, 1, qk_rope_head_dim, dtype=dtype, device=device)
        for s_len in batch_spec.seq_lens
    ]
    kv_cache = create_and_prepopulate_kv_cache(
        kv_c_contexts=kv_c_contexts,
        k_pe_contexts=k_pe_contexts,
        block_size=vllm_config.cache_config.block_size,
        head_size=head_size,
        dtype=dtype,
        device=device,
        num_blocks=vllm_config.cache_config.num_gpu_blocks,
        common_attn_metadata=common_attn_metadata,
        randomize_blocks=False,
        kv_cache_dtype="auto",
    )

    prefill_backend = get_mla_prefill_backend(vllm_config)(
        num_heads=num_heads,
        scale=1.0 / math.sqrt(head_size),
        kv_lora_rank=kv_lora_rank,
        qk_nope_head_dim=qk_nope_head_dim,
        qk_rope_head_dim=qk_rope_head_dim,
        v_head_dim=v_head_dim,
        vllm_config=vllm_config,
    )
    vllm_config.compilation_config.static_forward_context["placeholder"] = (
        SimpleNamespace(prefill_backend=prefill_backend)
    )

    builder_cls = FlashMLASparseBackend.get_builder_cls()
    builder = builder_cls(kv_cache_spec, ["placeholder"], vllm_config, device)
    metadata = builder.build(
        common_prefix_len=0, common_attn_metadata=common_attn_metadata
    )
    assert isinstance(metadata.prefill, SparseMLAPrefillMetadata)

    # Per-token sparse indices bounded by each token's position, with unique
    # offsets and -1 padding (same construction as the decode parity test).
    positions: list[int] = []
    for i in range(batch_spec.batch_size):
        ctx_len = batch_spec.seq_lens[i] - batch_spec.query_lens[i]
        positions.extend(ctx_len + q_idx for q_idx in range(batch_spec.query_lens[i]))
    sparse_indices = torch.empty(
        total_tokens, topk_tokens, dtype=torch.int32, device=device
    )
    for tok_idx in range(total_tokens):
        max_valid_idx = positions[tok_idx]
        offset = tok_idx * 7  # Prime number for varied offsets
        num_valid = min(topk_tokens // 2, max_valid_idx + 1)
        valid_range = torch.arange(num_valid, device=device, dtype=torch.int32)
        sparse_indices[tok_idx] = torch.cat(
            [
                (valid_range + offset) % (max_valid_idx + 1),
                torch.full(
                    (topk_tokens - num_valid,), -1, device=device, dtype=torch.int32
                ),
            ]
        )

    q = torch.rand(total_tokens, num_heads, head_size, dtype=dtype, device=device)
    mock_indexer = SimpleNamespace(
        topk_indices_buffer=sparse_indices, topk_tokens=topk_tokens
    )

    impl_cls = FlashMLASparseBackend.get_impl_cls()
    with set_current_vllm_config(vllm_config):
        impl = impl_cls(
            num_heads=num_heads,
            head_size=head_size,
            scale=1.0 / math.sqrt(head_size),
            num_kv_heads=1,
            alibi_slopes=None,
            sliding_window=None,
            kv_cache_dtype="auto",
            logits_soft_cap=None,
            attn_type="decoder",
            kv_sharing_target_layer_name=None,
            q_lora_rank=None,
            kv_lora_rank=kv_lora_rank,
            qk_nope_head_dim=qk_nope_head_dim,
            qk_rope_head_dim=qk_rope_head_dim,
            qk_head_dim=qk_nope_head_dim + qk_rope_head_dim,
            v_head_dim=v_head_dim,
            kv_b_proj=None,
            indexer=mock_indexer,
        )
    assert isinstance(impl.index_group, HiSparseMLAIndexGroup)
    cache_handle = impl.index_group.cache(impl.index_group_index)
    blocks_per_request = cdiv(cache_handle.runtime.region_stride, block_size)
    num_hot_blocks = cache_handle.runtime.max_num_reqs * blocks_per_request
    hot_cache = torch.zeros(
        num_hot_blocks * block_size * head_size,
        dtype=dtype,
        device=device,
    ).view(torch.int8)
    cache_handle.runtime.bind_hot_cache(
        hot_cache,
        byte_offset=0,
        block_stride=block_size * head_size * dtype.itemsize,
        num_blocks=num_hot_blocks,
        block_size=block_size,
        block_table=torch.arange(num_hot_blocks, dtype=torch.int32, device=device).view(
            cache_handle.runtime.max_num_reqs, blocks_per_request
        ),
    )
    cache_handle.runtime.request_state_indices = torch.arange(
        cache_handle.runtime.max_num_reqs, dtype=torch.int32, device=device
    )
    impl.prepare_for_batch(metadata)

    # Device-resident reference: the whole batch converted against the full
    # block table and run as one kernel call over the GPU cache.
    ref_topk, ref_topk_length = triton_convert_req_index_to_global_index(
        metadata.req_id_per_token,
        metadata.block_table,
        sparse_indices,
        BLOCK_SIZE=metadata.block_size,
        NUM_TOPK_TOKENS=topk_tokens,
        return_valid_counts=True,
    )
    reference, _ = impl._bf16_flash_mla_kernel(q, kv_cache, ref_topk, ref_topk_length)

    # Host-resident pool with identical contents.
    kv_pool = kv_cache.squeeze(1).cpu().pin_memory()
    cache_handle.runtime.bind_source_cache(kv_pool)
    cache_handle.source_block_table = metadata.block_table

    staging_calls = []
    original_gather = cache_handle.runtime.gather_prefill_cache

    def spy_gather(self, kv, plan, resident_cache=None):
        staged = original_gather(kv, plan, resident_cache)
        staging_calls.append((plan, staged.shape))
        return staged

    cache_handle.runtime.gather_prefill_cache = MethodType(
        spy_gather, cache_handle.runtime
    )

    backend_output, _ = impl._forward_bf16_kv(
        q, kv_pool, sparse_indices, metadata, q.shape[1]
    )
    torch.accelerator.synchronize()

    # Only the prefill rows' blocks were staged: the decode rows' 2048-token
    # contexts (32 blocks each) must stay off the staging gather.
    assert len(staging_calls) == 1
    plan, staged_shape = staging_calls[0]
    assert plan is metadata.prefill.host_staging_plan
    prefill_blocks = cdiv(batch_spec.seq_lens[-1], block_size)
    assert staged_shape[0] <= prefill_blocks + 1  # +1: block-0 tail padding

    torch.testing.assert_close(backend_output, reference, rtol=0.01, atol=0.01)


def test_hisparse_prefill_staging_remap():
    """Compacted staging references the same host rows as direct indexing."""
    block_size = 4
    block_table = torch.tensor(
        [[5, 2, -1, -1], [2, 7, 3, -1], [9, -1, -1, -1]], dtype=torch.int32
    )

    new_bt, row_ids = hisparse_prefill_staging_remap(block_table, block_size)

    assert new_bt.shape == block_table.shape
    assert row_ids.dtype == torch.int32 and new_bt.dtype == torch.int32
    n_unique = len({0, 2, 3, 5, 7, 9})  # -1 clamps to block 0
    flat_rows = row_ids.flatten()
    assert (flat_rows >= 0).sum() == n_unique * block_size
    for i in range(block_table.shape[0]):
        for j in range(block_table.shape[1]):
            orig = max(int(block_table[i, j]), 0)
            staged = int(new_bt[i, j])
            for k in range(block_size):
                assert int(flat_rows[staged * block_size + k]) == orig * block_size + k


def test_hisparse_prefill_staging_plan_masks_unused_blocks():
    block_table = torch.tensor([[5, 2, 7], [9, 3, 4]], dtype=torch.int32)
    plan = build_hisparse_prefill_staging_plan(
        block_table,
        seq_lens=torch.tensor([5, 8], dtype=torch.int32),
        block_size=4,
        staging_block_capacity=4,
    )

    assert (plan.block_table[:, 2] == 0).all()
    valid_rows = plan.row_ids[plan.row_ids >= 0]
    assert set((valid_rows[::4] // 4).tolist()) == {0, 2, 3, 5, 9}
    for row, used in enumerate((2, 2)):
        for column in range(used):
            staged_block = int(plan.block_table[row, column])
            host_block = int(block_table[row, column])
            assert int(valid_rows[staged_block * 4]) == host_block * 4


def test_hisparse_prefill_staging_plan_resolves_resident_sources():
    """Per-page resident hits become device sources; misses stay host DMAs."""
    block_size = 4
    resident_block_size = 2
    block_table = torch.tensor([[5, 2, 0], [9, 3, 0]], dtype=torch.int32)
    seq_lens = torch.tensor([5, 8], dtype=torch.int32)
    plan = build_hisparse_prefill_staging_plan(
        block_table, seq_lens, block_size, staging_block_capacity=4
    )
    # Two resident pages per host block; 0 entries are null (not resident).
    resident_table = torch.tensor(
        [[11, 12, 0, 13, 0, 0], [21, 0, 22, 23, 0, 0]], dtype=torch.int32
    )

    plan.ensure_gpu_sources(resident_table, resident_block_size)

    assert plan.gpu_row_ids is not None
    unique_hosts = (plan.row_ids[0].view(-1, block_size)[:, 0] // block_size).tolist()
    gpu_rows = plan.gpu_row_ids[0].view(-1, block_size)
    reps = {5: (0, 0), 2: (0, 1), 9: (1, 0), 3: (1, 1)}
    for u, host_id in enumerate(unique_hosts):
        for t in range(block_size):
            if host_id < 0:
                assert int(gpu_rows[u, t]) == -1
                continue
            if host_id == 0:
                assert int(gpu_rows[u, t]) == -1
                continue
            row, col = reps[host_id]
            page = t // resident_block_size
            res_block = int(resident_table[row, col * 2 + page])
            expected = (
                res_block * resident_block_size + t % resident_block_size
                if res_block > 0
                else -1
            )
            assert int(gpu_rows[u, t]) == expected
    valid_rows = plan.row_ids >= 0
    torch.testing.assert_close(
        plan.miss_mask,
        ((plan.gpu_row_ids < 0) & valid_rows).int(),
        check_dtype=False,
    )

    plan.ensure_gpu_sources(torch.zeros_like(resident_table), resident_block_size)

    assert plan.gpu_row_ids is not None
    assert (plan.gpu_row_ids == -1).all()


@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_hisparse_gather_prefill_cache_prefers_resident_rows():
    """Staged rows come from the resident cache when a shadow page exists."""
    if not _has_hisparse_ops():
        pytest.skip("hisparse CUDA ops unavailable")
    device = torch.device("cuda")
    block_size, resident_block_size, row_width = 4, 2, 16
    block_table = torch.tensor([[5, 2, 0], [9, 3, 0]], dtype=torch.int32, device=device)
    seq_lens = torch.tensor([5, 8], dtype=torch.int32, device=device)
    plan = build_hisparse_prefill_staging_plan(
        block_table, seq_lens, block_size, staging_block_capacity=4
    )
    resident_table = torch.tensor(
        [[11, 12, 0, 13, 0, 0], [21, 0, 22, 23, 0, 0]],
        dtype=torch.int32,
        device=device,
    )
    plan.ensure_gpu_sources(resident_table, resident_block_size)

    num_host_blocks, num_res_blocks = 10, 24
    host_cache = (
        torch.arange(num_host_blocks * block_size * row_width, dtype=torch.float32)
        .view(num_host_blocks, block_size, row_width)
        .pin_memory()
    )
    resident_cache = (
        (
            -torch.arange(
                num_res_blocks * resident_block_size * row_width, dtype=torch.float32
            )
            - 1.0
        )
        .view(num_res_blocks, resident_block_size, row_width)
        .to(device)
    )

    staged = HiSparseRuntime.gather_prefill_cache(
        None, host_cache, plan, resident_cache=resident_cache
    )

    staged_flat = staged.view(-1, row_width).cpu()
    host_flat = host_cache.view(-1, row_width)
    resident_flat = resident_cache.reshape(-1, row_width).cpu()
    gpu_rows = plan.gpu_row_ids[0].cpu()
    host_rows = plan.row_ids[0].cpu()
    for i in range(staged_flat.shape[0]):
        if int(host_rows[i]) < 0:
            assert int(plan.miss_mask[0, i]) == 0
            continue
        expected = (
            resident_flat[int(gpu_rows[i])]
            if int(gpu_rows[i]) >= 0
            else host_flat[int(host_rows[i])]
        )
        torch.testing.assert_close(staged_flat[i], expected)


def test_hisparse_fp8_decode_resolves_steps_then_runs_batched_attention(monkeypatch):
    """FP8 decode must use active dimensions when common metadata is padded."""
    device = torch.device("cpu")
    num_decodes = 2
    query_len = 3
    num_tokens = num_decodes * query_len
    q = torch.randn(num_tokens, 2, 4, device=device)
    topk = (
        torch.arange(num_tokens, dtype=torch.int32, device=device)
        .view(-1, 1)
        .expand(-1, 4)
    )
    steps: list[int] = []
    kernel_shapes: list[torch.Size] = []

    def swap_in(req_ids, *, logical_topk_indices, **kwargs):  # noqa: ARG001
        steps.append(int(logical_topk_indices[0, 0]))
        output = kwargs["attention_indices_out"]
        if output is not None:
            output.copy_(logical_topk_indices + 10)
        return topk[:num_decodes]

    def run_kernel(self, *, q, **kwargs):  # noqa: ARG001
        kernel_shapes.append(q.shape)
        return q[..., :1], None

    runtime = SimpleNamespace(
        hot=SimpleNamespace(attention_cache=torch.empty(1, device=device))
    )
    leader_cache = SimpleNamespace(
        runtime=runtime,
        source_block_table=torch.empty(
            num_decodes, 1, dtype=torch.int32, device=device
        ),
        swap_in=MagicMock(side_effect=swap_in),
    )
    follower_cache = SimpleNamespace(
        runtime=runtime,
        source_block_table=torch.empty(
            num_decodes, 1, dtype=torch.int32, device=device
        ),
        swap_in=MagicMock(side_effect=swap_in),
    )
    index_group = object.__new__(HiSparseMLAIndexGroup)
    index_group.caches = [leader_cache, follower_cache]
    index_group.physical_topk_indices = torch.empty(
        (num_tokens + 1, 4), dtype=torch.int32, device=device
    )
    index_group.valid_topk_counts = torch.empty(
        num_tokens + 1, dtype=torch.int32, device=device
    )
    index_group.request_ids = torch.arange(
        num_decodes, dtype=torch.int32, device=device
    )
    impl = SimpleNamespace(
        kv_lora_rank=1,
        index_group=index_group,
        index_group_index=0,
    )
    impl._fp8_flash_mla_kernel = MethodType(run_kernel, impl)
    metadata = SimpleNamespace(
        num_decodes=num_tokens,
        num_decode_tokens=num_tokens,
        decode_max_query_len=1,
        query_start_loc=torch.arange(
            0,
            num_tokens + 1,
            query_len,
            dtype=torch.int32,
            device=device,
        ),
        block_table=torch.empty(num_decodes, 1, dtype=torch.int32, device=device),
        block_size=64,
    )

    output = FlashMLASparseImpl._host_backed_fp8_decode(
        impl,
        q,
        topk,
        metadata,
        SimpleNamespace(),
        num_decodes,
        query_len,
    )

    assert steps == [0, 1, 2]
    assert kernel_shapes == [(num_decodes, query_len, 2, 4)]
    assert output.shape == (num_tokens, 2, 1)
    torch.testing.assert_close(
        index_group.physical_topk_indices[:num_tokens], topk + 10
    )

    follower_indices = index_group.convert_decode_logical_to_physical_topk(
        1,
        topk,
        metadata,
        return_valid_counts=False,
        num_decodes=num_decodes,
        decode_query_len=query_len,
    )
    assert steps == [0, 1, 2, 0, 1, 2]
    torch.testing.assert_close(follower_indices, topk + 10)


def test_hisparse_varlen_decode_uses_canonical_request_rows():
    """Graph metadata's query upper bound must preserve packed token rows."""
    num_decodes = 2
    topk = torch.arange(18, dtype=torch.int32).view(9, 2)

    def swap_in(_request_ids, *, logical_topk_indices, **_kwargs):
        return logical_topk_indices + 10

    cache = SimpleNamespace(
        source_block_table=torch.zeros((num_decodes, 1), dtype=torch.int32),
        swap_in=MagicMock(side_effect=swap_in),
    )
    index_group = object.__new__(HiSparseMLAIndexGroup)
    index_group.caches = [cache]
    index_group.physical_topk_indices = torch.empty((10, 2), dtype=torch.int32)
    index_group.valid_topk_counts = torch.empty(10, dtype=torch.int32)
    index_group.request_ids = torch.arange(num_decodes, dtype=torch.int32)
    metadata = SimpleNamespace(
        num_decodes=num_decodes,
        num_decode_tokens=topk.shape[0],
        decode_max_query_len=15,
        query_start_loc=torch.tensor([0, 8, 9], dtype=torch.int32),
        block_size=64,
    )

    physical_topk = index_group.convert_decode_logical_to_physical_topk(
        0,
        topk,
        metadata,
        return_valid_counts=False,
    )

    torch.testing.assert_close(physical_topk, topk + 10)
    request_rows = cache.swap_in.call_args_list[0].args[0]
    torch.testing.assert_close(request_rows, index_group.request_ids)


def test_flashinfer_hisparse_decode_runs_batched_attention():
    device = torch.device("cpu")
    num_tokens = 6
    q = torch.randn(num_tokens, 2, 4, device=device)
    topk = torch.zeros(num_tokens, 4, dtype=torch.int32, device=device)
    physical_topk = topk + 10
    valid_counts = torch.full((num_tokens,), 4, dtype=torch.int32, device=device)
    kernel_shapes: list[torch.Size] = []

    def convert_decode(self, *args, **kwargs):  # noqa: ARG001
        return physical_topk, valid_counts

    def prepare_kernel(self, *args, **kwargs):  # noqa: ARG001
        pass

    def run_kernel(self, q, cache, indices, counts):  # noqa: ARG001
        kernel_shapes.append(q.shape)
        return q[..., :1], None

    cache_handle = SimpleNamespace(
        runtime=SimpleNamespace(
            hot=SimpleNamespace(attention_cache=torch.empty(1, device=device))
        ),
    )
    index_group = object.__new__(HiSparseMLAIndexGroup)
    index_group.caches = [cache_handle]
    index_group.convert_decode_logical_to_physical_topk = MethodType(
        convert_decode, index_group
    )
    impl = object.__new__(FlashInferMLASparseImpl)
    impl.topk_indices_buffer = topk
    impl.index_group = index_group
    impl.index_group_index = 0
    impl._prepare_mqa_kernel = MethodType(prepare_kernel, impl)
    impl._run_mqa_kernel = MethodType(run_kernel, impl)
    metadata = SimpleNamespace(num_decode_tokens=num_tokens)

    output, lse = FlashInferMLASparseImpl.forward_mqa(
        impl,
        q,
        torch.empty(1, device=device),
        metadata,
        SimpleNamespace(),
    )

    assert kernel_shapes == [q.shape]
    assert output.shape == (num_tokens, 2, 1)
    assert lse is None


def test_flashattn_hisparse_decode_uses_index_group():
    num_tokens = 4
    q_nope = torch.empty(num_tokens, 2, 3, device=DEVICE_TYPE)
    q_rope = torch.empty(num_tokens, 2, 1, device=DEVICE_TYPE)
    topk = torch.zeros(num_tokens, 4, dtype=torch.int32, device=DEVICE_TYPE)
    physical = topk.clone()
    counts = torch.full((num_tokens,), 4, dtype=torch.int32, device=DEVICE_TYPE)
    index_group = object.__new__(HiSparseMLAIndexGroup)
    index_group.convert_decode_logical_to_physical_topk = MagicMock(
        return_value=(physical, counts)
    )
    index_group.physical_kv_cache = MagicMock(
        return_value=torch.empty(1, device=DEVICE_TYPE)
    )
    impl = object.__new__(FlashAttnMLASparseImpl)
    impl.topk_indices_buffer = topk
    impl.index_group = index_group
    impl.index_group_index = 0
    impl._run_mqa_kernel = MagicMock(return_value=q_nope[..., :1])
    metadata = SimpleNamespace(num_decode_tokens=num_tokens, block_size=64)

    output, lse = FlashAttnMLASparseImpl.forward_mqa(
        impl,
        (q_nope, q_rope),
        torch.empty(1, device=DEVICE_TYPE),
        metadata,
        SimpleNamespace(),
    )

    assert output.shape == (num_tokens, 2, 1)
    assert lse is None
    index_group.convert_decode_logical_to_physical_topk.assert_called_once()
    impl._run_mqa_kernel.assert_called_once()


def test_flashinfer_sm120_hisparse_decode_uses_index_group():
    num_tokens = 4
    q = torch.empty(num_tokens, 2, 4, device=DEVICE_TYPE)
    topk = torch.zeros(num_tokens, 4, dtype=torch.int32, device=DEVICE_TYPE)
    physical = topk.clone()
    index_group = object.__new__(HiSparseMLAIndexGroup)
    index_group.convert_decode_logical_to_physical_topk = MagicMock(
        return_value=physical
    )
    index_group.physical_kv_cache = MagicMock(
        return_value=torch.empty(1, device=DEVICE_TYPE)
    )
    impl = object.__new__(FlashInferMLASparseSM120Impl)
    impl.topk_indices_buffer = topk
    impl.index_group = index_group
    impl.index_group_index = 0
    impl._run_mqa_kernel = MagicMock(return_value=q[..., :1])
    metadata = SimpleNamespace(
        num_decode_tokens=num_tokens,
        topk_tokens=topk.shape[1],
    )

    output, lse = FlashInferMLASparseSM120Impl.forward_mqa(
        impl,
        q,
        torch.empty(1, device=DEVICE_TYPE),
        metadata,
        SimpleNamespace(),
    )

    assert output.shape == (num_tokens, 2, 1)
    assert lse is None
    index_group.convert_decode_logical_to_physical_topk.assert_called_once()
    impl._run_mqa_kernel.assert_called_once()


def test_hisparse_resident_prefill_uses_attention_block_stride():
    expected = torch.tensor([[19]], dtype=torch.int32)
    cache_handle = SimpleNamespace(
        all_context_pages_resident=True,
        view=SimpleNamespace(block_size=64, attention_block_stride=832),
        block_table=torch.tensor([[3]], dtype=torch.int32),
    )
    index_group = object.__new__(HiSparseMLAIndexGroup)
    index_group.caches = [cache_handle]
    index_group.physical_topk_indices = torch.empty((1, 1), dtype=torch.int32)
    index_group._convert_once = MagicMock(return_value=expected)
    topk = torch.zeros((2, 1), dtype=torch.int32)
    metadata = SimpleNamespace(
        req_id_per_token=torch.zeros(2, dtype=torch.int32),
    )

    result = index_group.convert_logical_to_physical_topk(
        0,
        topk,
        metadata,
        block_stride_rows=None,
        return_valid_counts=False,
    )

    assert result is expected
    assert index_group._convert_once.call_args.kwargs["block_stride_rows"] == 832


def test_hisparse_shared_sparse_builder_routes_multi_token_chunks_to_prefill():
    """Default sparse backends must route multi-token chunks to prefill."""
    builder = object.__new__(FlashInferMLASparseMetadataBuilder)
    builder.vllm_config = SimpleNamespace(
        attention_config=SimpleNamespace(hisparse_config=object()),
        speculative_config=SimpleNamespace(
            num_speculative_tokens=4,
            parallel_drafting=False,
        ),
        parallel_config=SimpleNamespace(decode_context_parallel_size=1),
    )

    builder._init_reorder_batch_threshold(
        1024,
        supports_spec_as_decode=True,
        supports_dcp_with_varlen=True,
    )

    assert builder.reorder_batch_threshold == 1


def test_hisparse_flashmla_reorders_full_speculative_window_as_decode():
    builder = object.__new__(FlashMLASparseMetadataBuilder)
    builder.vllm_config = SimpleNamespace(
        attention_config=SimpleNamespace(hisparse_config=object()),
        speculative_config=SimpleNamespace(
            num_speculative_tokens=3,
            parallel_drafting=False,
        ),
        parallel_config=SimpleNamespace(decode_context_parallel_size=1),
    )

    builder._init_reorder_batch_threshold(
        256,
        supports_spec_as_decode=True,
        supports_dcp_with_varlen=True,
    )

    assert builder.reorder_batch_threshold == 4


def test_hisparse_mixed_single_token_batch_is_not_decode_only():
    cache = SimpleNamespace(
        runtime=SimpleNamespace(
            begin_forward=lambda: None,
            eager_host_mirror=True,
        ),
    )
    metadata = SimpleNamespace(
        num_decode_tokens=1,
        num_actual_tokens=2,
        max_query_len=1,
        num_reqs=2,
        req_id_per_token=torch.arange(2),
    )

    HiSparseCacheHandle._prepare_for_batch(cache, metadata)

    assert not cache.decode_batch


def test_flashmla_cache_dtype_aliases_use_ds_layout():
    # kv-cache dtype aliases are canonicalized to fp8_ds_mla before the layer
    # stores kv_cache_dtype, so they cannot bypass the gate.
    for alias in ("fp8", "fp8_e4m3"):
        assert (
            _canonicalize_sparse_mla_kv_cache_dtype(FlashMLASparseBackend, alias)
            == "fp8_ds_mla"
        )


def test_flashmla_common_metadata_requires_uniform_decodes():
    common_metadata = SimpleNamespace(
        max_query_len=3,
        num_reqs=3,
        num_actual_tokens=6,
        query_start_loc_cpu=torch.tensor([0, 1, 3, 6]),
        is_prefilling=None,
    )

    split = split_decodes_and_prefills(
        common_metadata,
        decode_threshold=128,
        require_uniform=FlashMLASparseMetadataBuilder.require_uniform_decodes,
    )

    assert split == (1, 2, 1, 5)


def test_flashmla_fp8_metadata_excludes_zero_token_decode_padding(monkeypatch):
    monkeypatch.setattr(
        "vllm.v1.attention.backends.mla.flashmla_sparse.get_mla_metadata",
        lambda: (object(), None),
    )
    builder = SimpleNamespace(
        device=torch.device(DEVICE_TYPE),
        dummy_block_table=torch.zeros(7, 1, device=DEVICE_TYPE),
        max_model_len_tensor=torch.zeros(7, device=DEVICE_TYPE),
        pcp_dcp_kv_gather=False,
    )
    query_start_loc_cpu = torch.tensor([0, 110, 220, 330, 440, 550, 660, 660])
    common_metadata = SimpleNamespace(
        num_actual_tokens=660,
        query_start_loc_cpu=query_start_loc_cpu,
        seq_lens=torch.arange(7, device=DEVICE_TYPE),
    )
    metadata = FlashMLASparseMetadata(
        num_reqs=7,
        max_query_len=110,
        max_seq_len=110,
        num_actual_tokens=660,
        query_start_loc=query_start_loc_cpu.to(DEVICE_TYPE),
        slot_mapping=torch.arange(660, device=DEVICE_TYPE),
        block_table=torch.zeros(7, 1, dtype=torch.int32, device=DEVICE_TYPE),
        req_id_per_token=torch.zeros(660, dtype=torch.int32, device=DEVICE_TYPE),
        num_decodes=7,
        num_prefills=0,
        num_decode_tokens=660,
    )

    fp8_metadata = FlashMLASparseMetadataBuilder._build_fp8_separate_prefill_decode(
        builder, common_metadata, metadata
    )

    assert fp8_metadata.num_decodes == 6
    assert fp8_metadata.num_decode_tokens == 660
    assert fp8_metadata.decode is not None
    assert fp8_metadata.decode.decode_query_len == 110
    torch.testing.assert_close(
        fp8_metadata.decode.seq_lens, torch.arange(6, device=DEVICE_TYPE)
    )


@pytest.mark.parametrize("use_mixed_batch", [False, True])
def test_flashmla_fp8_paths_accept_decode_subset(monkeypatch, use_mixed_batch: bool):
    num_decode_tokens = 2
    num_batch_tokens = 5
    q = torch.empty(num_decode_tokens, 2, 3, device=DEVICE_TYPE)
    topk_indices = torch.empty(num_decode_tokens, 4, device=DEVICE_TYPE)
    kernel_q_shapes = []

    def convert_indices(*args, **kwargs):  # noqa: ARG001
        assert not kwargs.get("HAS_PREFILL_WORKSPACE", False)
        if not kwargs.get("return_valid_counts", False):
            return topk_indices
        valid_counts = torch.full(
            (num_decode_tokens,), 4, dtype=torch.int32, device=DEVICE_TYPE
        )
        return topk_indices, valid_counts

    monkeypatch.setattr(
        "vllm.v1.attention.backends.mla.flashmla_sparse."
        "triton_convert_req_index_to_global_index",
        convert_indices,
    )

    def run_kernel(**kwargs):
        kernel_q_shapes.append(kwargs["q"].shape)
        return kwargs["q"][..., :1], None

    if use_mixed_batch:
        fp8_metadata = FlashMLASparseMetadata.FP8KernelMetadata(
            scheduler_metadata=object(),  # type: ignore[arg-type]
            dummy_block_table=torch.empty(1, 1, dtype=torch.int32, device=DEVICE_TYPE),
            cache_lens=torch.empty(1, dtype=torch.int32, device=DEVICE_TYPE),
        )
    else:
        FP8Meta = FlashMLASparseMetadata.FP8SeparatePrefillDecode
        fp8_metadata = FP8Meta(
            num_decodes=1,
            num_prefills=1,
            num_decode_tokens=num_decode_tokens,
            num_prefill_tokens=num_batch_tokens - num_decode_tokens,
            decode=FP8Meta.Decode(
                seq_lens=torch.empty(1, dtype=torch.int32, device=DEVICE_TYPE),
                kernel_metadata=object(),  # type: ignore[arg-type]
                decode_query_len=num_decode_tokens,
            ),
            prefill=FP8Meta.Prefill(
                request_ids=torch.empty(
                    num_batch_tokens, dtype=torch.int32, device=DEVICE_TYPE
                ),
                workspace_starts=torch.empty(1, dtype=torch.int32, device=DEVICE_TYPE),
                chunks=[],
            ),
        )
    metadata = SimpleNamespace(
        fp8_extra_metadata=fp8_metadata,
        fp8_use_mixed_batch=use_mixed_batch,
        num_actual_tokens=num_batch_tokens,
        num_decode_tokens=num_decode_tokens,
        req_id_per_token=torch.empty(
            num_batch_tokens, dtype=torch.int32, device=DEVICE_TYPE
        ),
        block_table=torch.empty(1, 1, dtype=torch.int32, device=DEVICE_TYPE),
        block_size=64,
    )
    impl = SimpleNamespace(
        kv_cache_dtype="fp8_ds_mla",
        topk_indices_buffer=topk_indices,
        num_heads=2,
        kv_lora_rank=1,
        index_group=None,
        index_group_index=0,
        dcp_world_size=1,
        pcp_dcp_kv_gather=False,
        need_to_return_lse_for_decode=False,
        _fp8_flash_mla_kernel=run_kernel,
        _convert_logical_to_physical_topk=(
            lambda indices, metadata, **kwargs: convert_indices(
                metadata.req_id_per_token[: indices.shape[0]],
                metadata.block_table,
                indices,
                return_valid_counts=kwargs["return_valid_counts"],
            )
        ),
    )
    impl._forward_fp8_kv_mixed_batch = MethodType(
        FlashMLASparseImpl._forward_fp8_kv_mixed_batch, impl
    )
    impl._forward_fp8_kv_separate_prefill_decode = MethodType(
        FlashMLASparseImpl._forward_fp8_kv_separate_prefill_decode, impl
    )

    output, lse = FlashMLASparseImpl.forward_mqa(
        impl,
        q,
        torch.empty(0, device=DEVICE_TYPE),
        metadata,
        None,
    )

    assert kernel_q_shapes == [(1, num_decode_tokens, 2, 3)]
    assert output.shape == (num_decode_tokens, 2, 1)
    assert lse is None


def _build_sparse_dcp_vllm_config(
    local_heads: int,
    dcp_world_size: int,
    comm_backend: str = "ag_rs",
):
    """Minimal sparse-MLA VllmConfig for the FlashMLASparse DCP head-envelope
    guard. TP is simulated by mocking ``get_num_attention_heads`` to return the
    per-rank head count, as the decode-correctness test above does.
    """
    kv_lora_rank = 512
    qk_nope_head_dim = 128
    qk_rope_head_dim = 64
    v_head_dim = 128
    head_size = kv_lora_rank + qk_rope_head_dim
    topk_tokens = 128

    vllm_config = create_vllm_config(
        model_name="deepseek-ai/DeepSeek-V2-Lite-Chat",
        tensor_parallel_size=1,
        max_model_len=4096,
        block_size=64,
        hf_config_override={
            "index_topk": topk_tokens,
            "attn_module_list_cfg": [{"topk_tokens": topk_tokens}],
        },
    )
    model_config = vllm_config.model_config
    model_config.dtype = torch.bfloat16
    model_config.hf_text_config = SimpleNamespace(
        index_topk=topk_tokens,
        q_lora_rank=None,
        kv_lora_rank=kv_lora_rank,
        qk_nope_head_dim=qk_nope_head_dim,
        qk_rope_head_dim=qk_rope_head_dim,
        v_head_dim=v_head_dim,
        model_type="deepseek_v2",
    )
    del model_config.hf_config.index_topk  # Composite configs only nest this field.
    model_config.get_num_attention_heads = MethodType(
        lambda self, parallel_config: local_heads, model_config
    )
    model_config.get_num_kv_heads = MethodType(
        lambda self, parallel_config: 1, model_config
    )
    model_config.get_head_size = MethodType(lambda self: head_size, model_config)
    model_config.get_sliding_window = MethodType(lambda self: None, model_config)

    vllm_config.cache_config.cache_dtype = "fp8_ds_mla"
    vllm_config.parallel_config.decode_context_parallel_size = dcp_world_size
    vllm_config.parallel_config.dcp_comm_backend = comm_backend
    # The base builder clones the layer's dense-MHA prefill backend from
    # static_forward_context; the guard tests never run prefill.
    vllm_config.compilation_config.static_forward_context["placeholder"] = (
        SimpleNamespace(prefill_backend=None)
    )
    return vllm_config


@pytest.mark.skipif(
    torch.cuda.get_device_capability() < (9, 0),
    reason="FlashMLASparseBackend requires CUDA 9.0 or higher",
)
@pytest.mark.parametrize(
    "local_heads,dcp_world_size,should_raise",
    [
        (16, 8, True),
        (24, 4, True),
        (16, 4, False),
        (16, 1, False),
    ],
)
def test_fp8_dcp_head_envelope_guard(local_heads, dcp_world_size, should_raise):
    """The fp8 decode envelope (head padding + tile-scheduler metadata) is
    sized from the local head count while the kernel runs on the DCP-gathered
    heads, so the builder must reject configs where the two pad differently.
    """
    device = torch.device(DEVICE_TYPE)
    vllm_config = _build_sparse_dcp_vllm_config(local_heads, dcp_world_size)
    kv_cache_spec = create_standard_kv_cache_spec(vllm_config)
    builder_cls = FlashMLASparseBackend.get_builder_cls()

    if should_raise:
        with pytest.raises(NotImplementedError, match="envelope"):
            builder_cls(kv_cache_spec, ["placeholder"], vllm_config, device)
    else:
        builder = builder_cls(kv_cache_spec, ["placeholder"], vllm_config, device)
        gathered_heads = local_heads * dcp_world_size
        local_pad = 64 if local_heads <= 64 else 128
        gathered_pad = 64 if gathered_heads <= 64 else 128
        assert builder.fp8_decode_padded_heads == local_pad
        assert local_pad == gathered_pad


def test_fp8_mixed_batch_dcp_neutralizes_empty_rows(monkeypatch):
    """A decode row whose top-k shard holds no local candidates (all -1) has
    undefined kernel out/lse; it must come back as (0, -inf), the identity of
    the cross-rank LSE merge, or a NaN would survive the merge even at zero
    weight (0 * NaN = NaN)."""
    num_tokens, num_heads, head_dim = 3, 2, 3
    q = torch.empty(num_tokens, num_heads, head_dim, device=DEVICE_TYPE)
    local_indices = torch.tensor(
        [[0, 1, -1, -1], [-1, -1, -1, -1], [2, -1, 3, -1]],
        dtype=torch.int32,
        device=DEVICE_TYPE,
    )

    monkeypatch.setattr(
        "vllm.v1.attention.backends.mla.flashmla_sparse."
        "triton_filter_and_convert_dcp_index",
        lambda *args, **kwargs: local_indices,
    )

    def run_kernel(**kwargs):
        out = torch.full(
            (1, num_tokens, num_heads, 1), float("nan"), device=DEVICE_TYPE
        )
        lse = torch.full((1, num_heads, num_tokens), float("nan"), device=DEVICE_TYPE)
        for token_id in (0, 2):  # rows with local candidates get real values
            out[0, token_id] = float(token_id + 1)
            lse[0, :, token_id] = float(token_id + 1)
        return out, lse

    metadata = SimpleNamespace(
        fp8_extra_metadata=FlashMLASparseMetadata.FP8KernelMetadata(
            scheduler_metadata=object(),  # type: ignore[arg-type]
            dummy_block_table=torch.empty(1, 1, dtype=torch.int32, device=DEVICE_TYPE),
            cache_lens=torch.empty(1, dtype=torch.int32, device=DEVICE_TYPE),
        ),
        req_id_per_token=torch.empty(num_tokens, dtype=torch.int32, device=DEVICE_TYPE),
        block_table=torch.empty(1, 1, dtype=torch.int32, device=DEVICE_TYPE),
        block_size=64,
        cp_kv_cache_interleave_size=1,
    )
    impl = SimpleNamespace(
        index_group=None,
        index_group_index=0,
        dcp_world_size=2,
        dcp_rank=0,
        need_to_return_lse_for_decode=True,
        _fp8_flash_mla_kernel=run_kernel,
    )

    out, lse = FlashMLASparseImpl._forward_fp8_kv_mixed_batch(
        impl, q, torch.empty(0, device=DEVICE_TYPE), local_indices, metadata
    )

    assert torch.equal(out[1], torch.zeros_like(out[1]))
    assert torch.isneginf(lse[1]).all()
    for token_id in (0, 2):
        assert torch.equal(out[token_id], torch.full_like(out[token_id], token_id + 1))
        assert torch.equal(lse[token_id], torch.full_like(lse[token_id], token_id + 1))
    assert out.is_contiguous()
    assert not out.isnan().any()
    assert not lse.isnan().any()


def test_hisparse_prefill_reuses_builder_staging_plan():
    """Every layer must reuse the batch plan instead of synchronizing to dedupe."""
    plan = SimpleNamespace(
        block_table=torch.tensor([[0]], dtype=torch.int32),
        ensure_gpu_sources=MagicMock(),
    )
    staged = torch.empty((1, 1, 8))
    calls = []

    def gather(kv_cache, staging_plan, resident_cache=None):
        calls.append((kv_cache, staging_plan, resident_cache))
        return staged

    resident_cache = torch.empty((1, 1, 8))
    resident_block_table = torch.tensor([[1]], dtype=torch.int32)
    cache = SimpleNamespace(
        runtime=SimpleNamespace(
            gather_prefill_cache=gather,
        ),
        view=SimpleNamespace(cache=resident_cache, block_size=1),
        block_table=resident_block_table,
    )
    index_group = object.__new__(HiSparseMLAIndexGroup)
    index_group.caches = [cache]
    source = torch.empty((1, 1, 8))
    metadata = SimpleNamespace(
        num_decodes=0,
        num_decode_tokens=0,
        seq_lens=torch.tensor([1], dtype=torch.int32),
        prefill=SimpleNamespace(host_staging_plan=plan),
        req_id_per_token=torch.tensor([0], dtype=torch.int32),
    )

    result, block_table, request_ids = index_group.stage_prefill_rows(
        0, source, metadata
    )

    assert result is staged
    assert block_table is plan.block_table
    torch.testing.assert_close(request_ids, metadata.req_id_per_token)
    plan.ensure_gpu_sources.assert_called_once()
    args = plan.ensure_gpu_sources.call_args.args
    torch.testing.assert_close(args[0], resident_block_table)
    assert args[1] == 1
    assert calls == [(source, plan, resident_cache)]


def test_hisparse_fp8_prefill_gather_uses_dedicated_stream(monkeypatch):
    compute_stream = object()
    prefill_stream = MagicMock()
    gather = MagicMock()
    monkeypatch.setattr(index_group_module, "current_stream", lambda: compute_stream)
    monkeypatch.setattr(
        index_group_module.ops, "cp_gather_and_upconvert_fp8_kv_cache", gather
    )

    resident_cache = torch.empty((2, 4, 656), dtype=torch.uint8)
    ready = MagicMock()
    cache = SimpleNamespace(
        view=SimpleNamespace(cache=resident_cache, block_size=4),
        block_table=torch.tensor([[2, 3]], dtype=torch.int32),
    )
    index_group = object.__new__(HiSparseMLAIndexGroup)
    index_group.caches = [cache]
    index_group.prefill_stream = prefill_stream
    index_group.prefill_ready_events = [ready]
    source = torch.empty((3, 4, 656), dtype=torch.uint8)
    dst = torch.empty((8, 576), dtype=torch.bfloat16)
    block_table = torch.tensor([[0, 1]], dtype=torch.int32)
    workspace_starts = torch.tensor([0], dtype=torch.int32)
    plan = SimpleNamespace(
        block_table=torch.tensor([[0, 1]], dtype=torch.int32),
        row_ids=torch.arange(8, dtype=torch.int32).view(1, -1),
        gpu_row_ids=torch.full((1, 8), -1, dtype=torch.int32),
        ensure_gpu_sources=MagicMock(),
    )
    metadata = SimpleNamespace(
        prefill=SimpleNamespace(host_staging_plan=plan),
        num_decodes=0,
    )

    result = index_group.gather_fp8_prefill(
        0,
        source,
        dst,
        block_table,
        workspace_starts,
        1,
        metadata,
        0,
    )

    prefill_stream.wait_stream.assert_called_once_with(compute_stream)
    ready.record.assert_called_once_with(prefill_stream)
    assert result is ready
    gather.assert_called_once()
    args = gather.call_args
    assert args.args[0] is resident_cache
    assert args.args[1] is dst
    torch.testing.assert_close(args.args[2], plan.block_table)
    assert args.args[3] is workspace_starts
    assert args.args[4] == 1
    assert args.kwargs["host_cache"].data_ptr() == source.data_ptr()
    assert args.kwargs["host_row_ids"] is plan.row_ids
    assert args.kwargs["device_row_ids"] is plan.gpu_row_ids
    plan.ensure_gpu_sources.assert_called_once()
    ensure_args = plan.ensure_gpu_sources.call_args.args
    torch.testing.assert_close(ensure_args[0], cache.block_table)
    assert ensure_args[1] == 4


def test_sparse_impl_observes_repointed_indexer_buffer():
    """The MTP proposer repoints the draft's indexer at the target model's buffer
    after the backend impl is built, so the impl must resolve the buffer per read.
    Snapshotting it in __init__ leaves the layer reading indices nothing writes."""
    impl = object.__new__(FlashInferMLASparseImpl)
    own = torch.zeros(4, 8, dtype=torch.int32)
    target = torch.ones(4, 8, dtype=torch.int32)
    indexer = SimpleNamespace(topk_indices_buffer=own)
    impl.init_topk_indices_buffer(indexer, None)

    assert impl.topk_indices_buffer is own

    indexer.topk_indices_buffer = target

    assert impl.topk_indices_buffer is target


def test_explicit_topk_buffer_supersedes_indexer():
    """Backbone skip-topk layers have no indexer, and the proposer also assigns the
    shared buffer directly onto draft submodules."""
    impl = object.__new__(FlashInferMLASparseImpl)
    indexer = SimpleNamespace(
        topk_indices_buffer=torch.zeros(2, 2, dtype=torch.int32),
    )
    impl.init_topk_indices_buffer(indexer, None)
    explicit = torch.ones(2, 2, dtype=torch.int32)

    impl.topk_indices_buffer = explicit

    assert impl.topk_indices_buffer is explicit


def test_sparse_mla_common_impl_resolves_buffer_lazily():
    """Guards the whole SparseMLACommonImpl family at once: a backend that
    snapshots the buffer instead silently loses MTP buffer sharing."""
    assert issubclass(SparseMLACommonImpl, SharedTopkIndicesBuffer)
    assert isinstance(SparseMLACommonImpl.topk_indices_buffer, property), (
        "topk_indices_buffer must stay a lazily-resolved property"
    )