vLLM/tests/v1/attention/test_sparse_mla_backends.py

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for the sparse MLA backends and utilities."""

import math
from collections import deque
from types import MethodType, SimpleNamespace
from unittest.mock import MagicMock

import pytest
import torch

from tests.v1.attention.test_mla_backends import (
    BATCH_SPECS,
    BatchSpec,
    MockSparseMLAAttentionLayer,
    create_and_prepopulate_kv_cache,
)
from tests.v1.attention.utils import (
    create_common_attn_metadata,
    create_standard_kv_cache_spec,
    create_vllm_config,
)
from vllm import _custom_ops as ops
from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphCapture
from vllm.config import (
    CUDAGraphMode,
    HiSparseConfig,
    SpeculativeConfig,
    set_current_vllm_config,
)
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.worker import (
    HiSparseConnectorWorker,
)
from vllm.forward_context import set_forward_context
from vllm.model_executor.layers.attention import mla_attention
from vllm.model_executor.layers.attention import (
    sparse_mla_attention as sparse_mla_module,
)
from vllm.model_executor.layers.attention.mla_attention import _use_masked_mha
from vllm.model_executor.layers.attention.sparse_mla_attention import (
    GLOBAL_TOPK_MASK_MAX_BYTES,
    SparseMLACommonImpl,
    SparseMLAPrefillMetadata,
    _is_masked_mha_available,
    _masked_mha_workspace_fits,
    _topk_mask_shape,
    _use_dense_mha_prefill,
)
from vllm.model_executor.layers.linear import ColumnParallelLinear
from vllm.platforms import current_platform
from vllm.platforms.interface import DeviceCapability

# This module's end-to-end sparse-backend correctness test supports CUDA and
# ROCm.  The remaining tests exercise CUDA-specific FlashMLA/FlashInfer APIs
# and are skipped by the autouse fixture below when collecting on ROCm.
if not (current_platform.is_cuda() or current_platform.is_rocm()):
    pytest.skip(
        "Sparse MLA backend tests require CUDA or ROCm.",
        allow_module_level=True,
    )

from vllm.model_executor.layers.attention.mla_attention import (
    _canonicalize_sparse_mla_kv_cache_dtype,
)
from vllm.model_executor.layers.attention.sparse_mla_attention import (
    SharedTopkIndicesBuffer,
)
from vllm.utils.math_utils import cdiv
from vllm.utils.torch_utils import current_stream
from vllm.v1.attention.backends.mla import index_group as index_group_module

if current_platform.is_cuda():
    from vllm.v1.attention.backends.mla import (
        flashattn_mla_sparse as flashattn_sparse_module,
    )
    from vllm.v1.attention.backends.mla.flashattn_mla_sparse import (
        FlashAttnMLASparseFA4Backend,
        FlashAttnMLASparseFA4Impl,
        FlashAttnMLASparseImpl,
        FlashAttnMLASparseMetadataBuilder,
        _fa4_cute_mla_available,
    )
from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import (
    FlashInferMLASparseImpl,
    FlashInferMLASparseTRTLLMBackend,
)
from vllm.v1.attention.backends.mla.flashinfer_mla_sparse_sm120 import (
    FlashInferMLASparseSM120Impl,
)
from vllm.v1.attention.backends.mla.flashmla_sparse import (
    FlashMLASparseBackend,
    FlashMLASparseImpl,
    FlashMLASparseMetadata,
    FlashMLASparseMetadataBuilder,
    triton_convert_req_index_to_global_index,
)
from vllm.v1.attention.backends.mla.index_group import (
    HiSparseMLAIndexGroup,
    SparseMLAIndexGroupBuilder,
)
from vllm.v1.attention.backends.mla.indexer import (
    DeepseekV32IndexerMetadataBuilder,
)
from vllm.v1.attention.backends.mla.prefill import get_mla_prefill_backend
from vllm.v1.attention.backends.utils import (
    split_decodes_and_prefills,
    split_prefill_chunks,
)
from vllm.v1.attention.ops import flashmla
from vllm.v1.hisparse import runtime as hisparse_runtime
from vllm.v1.hisparse.runtime import (
    HiSparseCacheHandle,
    HiSparseRuntime,
    ResolvedHiSparseConfig,
    _has_hisparse_ops,
    build_hisparse_prefill_staging_plan,
    hisparse_prefill_staging_remap,
)
from vllm.v1.hisparse.types import SparseKVRowMirror

if current_platform.is_rocm():
    from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import (
        ROCMAiterMLASparseBackend,
    )

    SPARSE_BACKENDS = [ROCMAiterMLASparseBackend]
    SPARSE_BACKEND_IDS = ["ROCMAiterMLA"]
    SPARSE_KV_CACHE_DTYPES = ["auto", "fp8"]
    SPARSE_BLOCK_SIZES = [64]
else:
    SPARSE_BACKENDS = [FlashMLASparseBackend, FlashInferMLASparseTRTLLMBackend]
    SPARSE_BACKEND_IDS = ["FlashMLA", "FlashInferTRTLLM"]
    SPARSE_KV_CACHE_DTYPES = ["auto", "fp8", "fp8_ds_mla", "nvfp4_ds_mla"]
    SPARSE_BLOCK_SIZES = [32, 64]


@pytest.fixture(autouse=True)
def _skip_cuda_specific_sparse_mla_tests_on_rocm(request):
    """Skip tests whose helpers exercise CUDA-only sparse MLA APIs."""
    rocm_portable_tests = {
        "test_sparse_backend_decode_correctness",
        "test_sparse_backend_prefill_correctness",
        "test_rocm_fused_v32_dense_prefill_matches_mqa",
        "test_rocm_long_bf16_prefill_avoids_nonpersistent_aiter",
    }
    if (
        current_platform.is_rocm()
        and getattr(request.node, "originalname", request.node.name)
        not in rocm_portable_tests
    ):
        pytest.skip("This sparse MLA test exercises a CUDA-specific API")


SPARSE_BACKEND_BATCH_SPECS = {
    name: BATCH_SPECS[name]
    for name in [
        "small_decode",
        "mixed_small",
        "mixed_medium",
        "small_prefill",
        "medium_prefill",
        "single_prefill",
    ]
}

SPARSE_BACKEND_BATCH_SPECS["large_q_prefill"] = BatchSpec(
    seq_lens=[1024] * 2, query_lens=[256] * 2
)
SPARSE_BACKEND_BATCH_SPECS["large_q_pure_prefill"] = BatchSpec(
    seq_lens=[256] * 2, query_lens=[256] * 2
)
SPARSE_BACKEND_BATCH_SPECS["large_decode_over_topk"] = BatchSpec(
    seq_lens=[256, 384], query_lens=[1, 1]
)

DEVICE_TYPE = current_platform.device_type


@pytest.mark.parametrize(
    ("batch_name", "tensor_parallel_size"),
    [
        ("large_q_prefill", 2),
        ("large_q_pure_prefill", 4),
        ("large_q_pure_prefill", 1),
    ],
)
def test_rocm_long_bf16_prefill_avoids_nonpersistent_aiter(
    default_vllm_config,
    dist_init,
    workspace_init,
    monkeypatch,
    batch_name,
    tensor_parallel_size,
):
    """MI300 long BF16 prefills must remain correct without AITER ASM dispatch."""
    if not current_platform.is_rocm():
        pytest.skip("ROCm sparse MLA regression")
    from vllm._aiter_ops import rocm_aiter_ops

    # Exercise the gfx942 routing decision on other ROCm devices too;
    # the numerical kernel still compiles for the actual device.
    monkeypatch.setattr("vllm.platforms.rocm.on_gfx942", lambda: True)

    def unsupported_aiter(*args, **kwargs):
        pytest.fail("Long gfx942 BF16 prefill reached nonpersistent AITER")

    monkeypatch.setattr(rocm_aiter_ops, "mla_decode_fwd", unsupported_aiter)
    test_sparse_backend_decode_correctness(
        default_vllm_config=default_vllm_config,
        dist_init=dist_init,
        backend_cls=ROCMAiterMLASparseBackend,
        batch_name=batch_name,
        kv_cache_dtype="auto",
        tensor_parallel_size=tensor_parallel_size,
        block_size=64,
        workspace_init=workspace_init,
        q_scale=1.0,
        k_scale=1.0,
        total_num_heads=128,
        qk_nope_head_dim=128,
        qk_rope_head_dim=64,
        v_head_dim=128,
        monkeypatch=monkeypatch,
    )


def test_nope_flashinfer_sparse_mla_uses_model_scale(monkeypatch):
    """Weight absorption must not change the model's attention temperature."""
    model_scale = 256**-0.5
    kv_lora_rank = 512
    topk = torch.zeros((1, 1), dtype=torch.int32)
    metadata = SimpleNamespace(
        req_id_per_token=torch.zeros(1, dtype=torch.int32),
        block_table=torch.zeros((1, 1), dtype=torch.int32),
        block_size=32,
        num_decode_tokens=1,
    )
    recorded_scale = None

    impl = object.__new__(FlashInferMLASparseImpl)
    impl.scale = model_scale
    impl.qk_nope_head_dim = 256
    impl.kv_lora_rank = kv_lora_rank
    impl.qk_rope_head_dim = 0
    impl.kv_cache_dtype = "auto"
    impl.topk_indices_buffer = topk
    impl.dcp_world_size = 1
    impl._workspace_buffer = torch.empty(1)
    impl.bmm1_scale = None
    impl.bmm2_scale = None
    impl.is_nope_mla = True
    impl.need_to_return_lse_for_decode = False
    impl.index_group = MagicMock()
    impl.index_group_index = 0
    impl.index_group.convert_logical_to_physical_topk.return_value = (
        topk,
        torch.ones(1, dtype=torch.int32),
    )

    import flashinfer.decode

    def fake_flashinfer(**kwargs):
        nonlocal recorded_scale
        recorded_scale = kwargs["bmm1_scale"]
        return torch.zeros((1, 1, 1, kv_lora_rank))

    monkeypatch.setattr(
        flashinfer.decode,
        "trtllm_batch_decode_with_kv_cache_mla",
        fake_flashinfer,
    )
    impl.forward_mqa(
        torch.zeros(1, 1, kv_lora_rank),
        torch.zeros(1, 32, kv_lora_rank),
        metadata,
        SimpleNamespace(),
    )

    assert recorded_scale == model_scale
    assert recorded_scale != kv_lora_rank**-0.5


def test_hisparse_routes_prefill_to_sparse_mqa():
    layer = SimpleNamespace(hisparse_cache=object())

    assert not mla_attention.MLAAttention._use_sparse_mha(layer, SimpleNamespace())


def test_hisparse_metadata_keeps_short_prefill_indexer_enabled():
    config = SimpleNamespace(
        attention_config=SimpleNamespace(
            hisparse_config=object(), sparse_mla_force_mqa=False
        )
    )

    assert not _use_dense_mha_prefill(config, prefill_max_seq_len=512, topk_tokens=2048)


def test_unified_mla_prepares_batch_without_slot_mapping(monkeypatch):
    layer = SimpleNamespace(
        prepare_kv_cache_update=MagicMock(),
        update_kv_cache=MagicMock(),
    )
    metadata = object()
    monkeypatch.setattr(
        mla_attention,
        "get_attention_context",
        lambda _: (metadata, layer, torch.empty(0), None),
    )
    tensor = torch.empty(0, device=DEVICE_TYPE)

    mla_attention.unified_mla_kv_cache_update(tensor, tensor, "layer", "auto", tensor)

    layer.prepare_kv_cache_update.assert_called_once_with(metadata)
    layer.update_kv_cache.assert_not_called()


def _float_to_e8m0_truncate(f: float) -> float:
    """Simulate SM100's float -> e8m0 -> bf16 scale conversion.
    e8m0 format only stores the exponent (power of 2).
    cudaRoundZero truncates toward zero, meaning we round down to the
    nearest power of 2.
    """
    if f <= 0:
        return 0.0
    # e8m0 = floor(log2(f)), then 2^(e8m0)
    # This is equivalent to truncating to the nearest power of 2 below f
    exp = math.floor(math.log2(f))
    return 2.0**exp


def _dequantize_fp8_ds_mla_entry(
    cache_slice: torch.Tensor,
    kv_lora_rank: int,
    rope_dim: int,
    dtype: torch.dtype,
    simulate_sm100_e8m0_scales: bool = False,
) -> tuple[torch.Tensor, torch.Tensor]:
    """Dequantize a single fp8_ds_mla cache entry back to latent + rope.

    Args:
        simulate_sm100_e8m0_scales: If True, simulate the SM100 kernel's
            float -> e8m0 -> bf16 scale conversion path.

    """
    # The first kv_lora_rank bytes store FP8 latent values with one scale per
    # 128 element tile written as float32 right after the latent payload.
    scales = cache_slice.view(torch.float32)[kv_lora_rank // 4 : kv_lora_rank // 4 + 4]
    latent = torch.empty(kv_lora_rank, dtype=torch.float16, device=cache_slice.device)
    for tile_idx in range(4):
        tile_start = tile_idx * 128
        tile_end = tile_start + 128
        scale_val = float(scales[tile_idx].item())
        if simulate_sm100_e8m0_scales:
            # Simulate the lossy float -> e8m0 -> bf16 conversion
            scale_val = _float_to_e8m0_truncate(scale_val)
        ops.convert_fp8(
            latent[tile_start:tile_end],
            cache_slice[tile_start:tile_end],
            scale_val,
            kv_dtype="fp8",
        )
    latent = latent.to(dtype)

    rope_offset = kv_lora_rank // 2 + 8
    rope_vals = cache_slice.view(dtype)[rope_offset : rope_offset + rope_dim]
    return latent, rope_vals.clone()


def _quantize_dequantize_fp8_ds_mla(
    kv_c: torch.Tensor,
    k_pe: torch.Tensor,
    block_size: int,
    scale: torch.Tensor,
    simulate_sm100_e8m0_scales: bool = False,
) -> tuple[torch.Tensor, torch.Tensor]:
    """Round-trip kv_c/k_pe though the fp8_ds_mla cache layout.

    Args:
        simulate_sm100_e8m0_scales: If True, simulate the SM100 kernel's
            float -> e8m0 -> bf16 scale conversion in dequantization.

    """
    if kv_c.numel() == 0:
        return kv_c.clone(), k_pe.clone()

    kv_lora_rank = kv_c.shape[-1]
    rope_dim = k_pe.shape[-1]
    num_tokens = kv_c.shape[0]
    num_blocks = max(1, math.ceil(num_tokens / block_size))
    # The 64-dim RoPE slot is always there; NoPE rows zero-fill it.
    entry_size = kv_lora_rank + 4 * 4 + 2 * 64

    tmp_cache = torch.zeros(
        num_blocks, block_size, entry_size, dtype=torch.uint8, device=kv_c.device
    )
    slot_mapping = torch.arange(num_tokens, dtype=torch.long, device=kv_c.device)

    ops.concat_and_cache_mla(
        kv_c, k_pe, tmp_cache, slot_mapping, kv_cache_dtype="fp8_ds_mla", scale=scale
    )

    dequant_kv_c = torch.empty_like(kv_c)
    dequant_k_pe = torch.empty_like(k_pe)

    for token_idx in range(num_tokens):
        slot = slot_mapping[token_idx].item()
        block_idx = slot // block_size
        block_offset = slot % block_size
        cache_slice = tmp_cache[block_idx, block_offset]
        latent, rope_vals = _dequantize_fp8_ds_mla_entry(
            cache_slice,
            kv_lora_rank,
            rope_dim,
            kv_c.dtype,
            simulate_sm100_e8m0_scales=simulate_sm100_e8m0_scales,
        )
        dequant_kv_c[token_idx] = latent
        dequant_k_pe[token_idx] = rope_vals

    return dequant_kv_c, dequant_k_pe


_E2M1_TABLE = [0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0]


def _quantize_dequantize_nvfp4_ds_mla(
    kv_c: torch.Tensor,
    k_pe: torch.Tensor,
    block_size: int,
    kv_cache_dtype: str,
    scale: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
    """Round-trip kv_c/k_pe through the nvfp4_ds_mla cache layout.

    Layout (per token, uint8): [e2m1-packed NoPE | unscaled e4m3 RoPE |
    per-16 e4m3 NoPE SFs]. The SF bytes are stored permuted (an 8x4 -> 4x8
    transpose: the scale for element block s lives at byte 8 * (s & 3) +
    (s >> 2)) so that the 8 scales one FlashMLA dequant thread needs are
    contiguous. Dequant: x = value * float(sf), which is exact in bf16, so
    this python dequant matches the kernels bit-for-bit.
    """
    if kv_c.numel() == 0:
        return kv_c.clone(), k_pe.clone()

    kv_lora_rank = kv_c.shape[-1]
    rope_dim = k_pe.shape[-1]
    num_tokens = kv_c.shape[0]
    num_blocks = max(1, math.ceil(num_tokens / block_size))
    entry_size = (
        math.ceil((kv_lora_rank // 2 + rope_dim + kv_lora_rank // 16) / 16) * 16
    )
    sf_nope_off = kv_lora_rank // 2 + rope_dim

    tmp_cache = torch.zeros(
        num_blocks, block_size, entry_size, dtype=torch.uint8, device=kv_c.device
    )
    slot_mapping = torch.arange(num_tokens, dtype=torch.long, device=kv_c.device)
    ops.concat_and_cache_mla(
        kv_c, k_pe, tmp_cache, slot_mapping, kv_cache_dtype=kv_cache_dtype, scale=scale
    )

    tokens = tmp_cache.view(-1, entry_size)[:num_tokens]
    table = torch.tensor(
        _E2M1_TABLE + [-v for v in _E2M1_TABLE],
        dtype=torch.float32,
        device=kv_c.device,
    )

    def unpack_e2m1(packed: torch.Tensor) -> torch.Tensor:
        lo = (packed & 0xF).long()
        hi = (packed >> 4).long()
        return table[torch.stack([lo, hi], dim=-1).flatten(-2)]

    nope_vals = unpack_e2m1(tokens[:, : kv_lora_rank // 2])
    num_nope_sf = kv_lora_rank // 16
    nope_sf = (
        # undo the on-wire SF permutation -> element-block order
        tokens[:, sf_nope_off : sf_nope_off + num_nope_sf]
        .unflatten(-1, (4, num_nope_sf // 4))
        .transpose(-1, -2)
        .flatten(-2)
        .view(torch.float8_e4m3fn)
        .float()
    )
    dequant_kv_c = (
        (nope_vals.unflatten(-1, (-1, 16)) * nope_sf.unsqueeze(-1)).flatten(-2)
    ).to(kv_c.dtype)

    # RoPE: plain e4m3, no scale factor
    rope_raw = tokens[:, kv_lora_rank // 2 : sf_nope_off]
    dequant_k_pe = rope_raw.view(torch.float8_e4m3fn).to(k_pe.dtype)

    return dequant_kv_c, dequant_k_pe


@pytest.mark.parametrize(
    "backend_cls",
    SPARSE_BACKENDS,
    ids=SPARSE_BACKEND_IDS,
)
@pytest.mark.parametrize("batch_name", list(SPARSE_BACKEND_BATCH_SPECS.keys()))
@pytest.mark.parametrize("kv_cache_dtype", SPARSE_KV_CACHE_DTYPES)
@pytest.mark.parametrize("tensor_parallel_size", [1, 2, 4])
@pytest.mark.parametrize("block_size", SPARSE_BLOCK_SIZES)
@pytest.mark.parametrize(("q_scale", "k_scale"), [(1.0, 1.0), (2.0, 3.0)])
@pytest.mark.parametrize(
    ("total_num_heads", "qk_nope_head_dim", "qk_rope_head_dim", "v_head_dim"),
    [
        pytest.param(128, 128, 64, 128, id="deepseek"),
        pytest.param(64, 256, 0, 256, id="glm53_flash_nope"),
    ],
)
def test_sparse_backend_decode_correctness(
    default_vllm_config,
    dist_init,
    backend_cls,
    batch_name,
    kv_cache_dtype,
    tensor_parallel_size,
    block_size,
    workspace_init,
    q_scale: float,
    k_scale: float,
    total_num_heads: int,
    qk_nope_head_dim: int,
    qk_rope_head_dim: int,
    v_head_dim: int,
    monkeypatch,
    block_stride_rows: int | None = None,
):
    if qk_rope_head_dim == 0 and (
        backend_cls != FlashMLASparseBackend
        or kv_cache_dtype not in ("auto", "fp8_ds_mla")
        or torch.cuda.get_device_capability()[0] != 9
    ):
        pytest.skip("NoPE sparse MLA is served by FlashMLA on SM90 only")
    if (
        batch_name == "large_q_pure_prefill"
        and backend_cls == FlashMLASparseBackend
        and kv_cache_dtype == "fp8_ds_mla"
        and tensor_parallel_size == 4
    ):
        monkeypatch.setattr(
            "vllm.v1.attention.backends.mla.flashmla_sparse.split_prefill_chunks",
            lambda rows, capacity: [(i, i + 1) for i in range(len(rows))],
        )
    if kv_cache_dtype not in backend_cls.supported_kv_cache_dtypes:
        pytest.skip(f"{backend_cls.get_name()} does not support {kv_cache_dtype}")

    if (
        backend_cls == FlashMLASparseBackend
        and kv_cache_dtype.startswith("fp8")
        and kv_cache_dtype != "fp8_ds_mla"
    ):
        pytest.skip(
            "FlashMLA Sparse Attention backend fp8 only supports "
            "fp8_ds_mla kv-cache dtype"
        )

    if kv_cache_dtype == "nvfp4_ds_mla":
        # Unlike "fp8" (an alias canonicalized to "fp8_ds_mla" above), the
        # NVFP4 DS-MLA dtype must reach the backend unchanged.
        assert (
            _canonicalize_sparse_mla_kv_cache_dtype(backend_cls, kv_cache_dtype)
            == kv_cache_dtype
        )
        device_capability = current_platform.get_device_capability()
        if device_capability is None or device_capability.major != 10:
            pytest.skip("The NVFP4 DS-MLA kv-cache dtype requires SM 10.x")

    if not backend_cls.supports_block_size(block_size):
        pytest.skip(
            f"{backend_cls.get_name()} does not support block_size={block_size}"
        )

    if backend_cls == FlashMLASparseBackend:
        ok, reason = flashmla.is_flashmla_sparse_supported()
        if not ok:
            pytest.skip(reason)
    elif backend_cls == FlashInferMLASparseTRTLLMBackend:
        device_capability = current_platform.get_device_capability()
        if device_capability is None or not backend_cls.supports_compute_capability(
            device_capability
        ):
            pytest.skip("FlashInferMLASparseTRTLLMBackend requires SM 10.x capability")

    batch_spec = SPARSE_BACKEND_BATCH_SPECS[batch_name]
    use_fp8_ds_mla_quantization = kv_cache_dtype == "fp8_ds_mla"
    use_nvfp4_ds_mla_quantization = kv_cache_dtype == "nvfp4_ds_mla"

    device = torch.device(DEVICE_TYPE)
    dtype = torch.bfloat16

    # Compute per-rank heads for simulated TP
    num_heads = max(1, total_num_heads // tensor_parallel_size)

    kv_lora_rank = 512
    head_size = kv_lora_rank + qk_rope_head_dim
    topk_tokens = 128

    max_seqlen = max(batch_spec.seq_lens)
    total_cache_tokens = sum(batch_spec.seq_lens)

    # Note: We use TP=1 to avoid multi-GPU requirements in CI.
    # The test simulates head partitioning via mocked methods below.
    vllm_config = create_vllm_config(
        model_name="deepseek-ai/DeepSeek-V2-Lite-Chat",
        tensor_parallel_size=1,
        max_model_len=max_seqlen,
        num_gpu_blocks=max(2048, cdiv(total_cache_tokens, block_size) + 1),
        block_size=block_size,
        hf_config_override={
            "index_topk": topk_tokens,
            "attn_module_list_cfg": [{"topk_tokens": topk_tokens}],
        },
    )
    # This shared correctness test constructs a sparse SDPA reference for every
    # query token. ROCm normally routes short prefills to dense MHA, whose
    # output intentionally differs from that sparse reference. Keep this test
    # on the sparse-MQA path; dense-prefill routing has dedicated ROCm tests.
    if current_platform.is_rocm():
        vllm_config.attention_config.sparse_mla_force_mqa = True
    model_config = vllm_config.model_config
    model_config.hf_text_config = SimpleNamespace(
        index_topk=topk_tokens,
        q_lora_rank=None,
        kv_lora_rank=kv_lora_rank,
        qk_nope_head_dim=qk_nope_head_dim,
        qk_rope_head_dim=qk_rope_head_dim,
        v_head_dim=v_head_dim,
        model_type="deepseek_v2",
    )
    del model_config.hf_config.index_topk  # Composite configs only nest this field.
    model_config.dtype = dtype
    model_config.get_num_attention_heads = MethodType(
        lambda self, parallel_config: num_heads,
        model_config,
    )
    model_config.get_num_kv_heads = MethodType(
        lambda self, parallel_config: 1, model_config
    )
    model_config.get_head_size = MethodType(lambda self: head_size, model_config)
    model_config.get_sliding_window = MethodType(lambda self: None, model_config)

    kv_cache_spec = create_standard_kv_cache_spec(vllm_config)

    torch.manual_seed(0)

    scale = 1.0 / math.sqrt(head_size)

    # Shared MLA projection weights to keep reference and backend in sync
    W_UK = torch.rand(
        kv_lora_rank, num_heads, qk_nope_head_dim, dtype=dtype, device=device
    )
    W_UV = torch.rand(kv_lora_rank, num_heads, v_head_dim, dtype=dtype, device=device)

    # Build synthetic decode-only workload
    seq_lens = batch_spec.seq_lens
    query_lens = batch_spec.query_lens

    # Pre-compute positions and sparse indices for all tokens.
    # We need these BEFORE computing the reference to use sparse attention masks.
    total_query_tokens = sum(query_lens)
    positions = []
    for i in range(batch_spec.batch_size):
        s_len = seq_lens[i]
        q_len = query_lens[i]
        ctx_len = s_len - q_len
        for q_idx in range(q_len):
            positions.append(ctx_len + q_idx)

    # Create sparse indices with UNIQUE per-token offsets to catch bugs where
    # the kernel uses wrong indices for some tokens (e.g., due to incorrect
    # tensor shapes like [1, num_tokens, ...] instead of [num_tokens, 1, ...]).
    # Also include -1 masked indices to verify the kernel handles them correctly.
    sparse_indices = torch.empty(
        total_query_tokens, topk_tokens, dtype=torch.int32, device=device
    )
    for tok_idx in range(total_query_tokens):
        max_valid_idx = positions[tok_idx]
        offset = tok_idx * 7  # Prime number for varied offsets
        # Use only half the topk indices as valid, mask the rest with -1
        # This tests that the kernel correctly ignores -1 indices
        num_valid = min(topk_tokens // 2, max_valid_idx + 1)
        if num_valid > 0:
            valid_range = torch.arange(num_valid, device=device, dtype=torch.int32)
            tok_indices = (valid_range + offset) % (max_valid_idx + 1)
            # Pad with -1 for the remaining positions
            tok_indices = torch.cat(
                [
                    tok_indices,
                    torch.full(
                        (topk_tokens - num_valid,), -1, device=device, dtype=torch.int32
                    ),
                ]
            )
        else:
            tok_indices = torch.full(
                (topk_tokens,), -1, device=device, dtype=torch.int32
            )
            tok_indices[0] = 0  # At least one valid index
        sparse_indices[tok_idx] = tok_indices

    all_q_vllm, all_kv_c_vllm, all_k_pe_vllm = [], [], []
    kv_c_contexts, k_pe_contexts = [], []
    reference_outputs = []

    kv_cache_scale = torch.tensor(k_scale, dtype=torch.float32, device=device)
    global_token_idx = 0

    for i in range(batch_spec.batch_size):
        s_len = seq_lens[i]
        q_len = query_lens[i]
        ctx_len = s_len - q_len

        q_c = torch.rand(
            q_len,
            num_heads,
            qk_nope_head_dim + qk_rope_head_dim,
            dtype=dtype,
            device=device,
        )
        kv_c_full = torch.rand(s_len, kv_lora_rank, dtype=dtype, device=device)
        k_pe_full = torch.rand(s_len, 1, qk_rope_head_dim, dtype=dtype, device=device)

        if use_fp8_ds_mla_quantization:
            is_sm100 = torch.cuda.get_device_capability()[0] >= 10
            kv_c_full, k_pe_squeezed = _quantize_dequantize_fp8_ds_mla(
                kv_c_full,
                k_pe_full.squeeze(1),
                block_size=block_size,
                scale=kv_cache_scale,
                simulate_sm100_e8m0_scales=is_sm100,
            )
            k_pe_full = k_pe_squeezed.unsqueeze(1)
        elif use_nvfp4_ds_mla_quantization:
            kv_c_full, k_pe_squeezed = _quantize_dequantize_nvfp4_ds_mla(
                kv_c_full,
                k_pe_full.squeeze(1),
                block_size=block_size,
                kv_cache_dtype=kv_cache_dtype,
                scale=kv_cache_scale,
            )
            k_pe_full = k_pe_squeezed.unsqueeze(1)

        q_nope, q_pe = q_c.split([qk_nope_head_dim, qk_rope_head_dim], dim=-1)
        ql_nope = torch.einsum("qnh,lnh->qnl", q_nope, W_UK)
        q_mqa = torch.cat([ql_nope, q_pe], dim=-1)

        k_mqa = torch.cat([kv_c_full, k_pe_full.squeeze(1)], dim=-1)
        v_mqa = kv_c_full

        # Compute sparse SDPA reference per query token using its sparse indices
        for q_idx in range(q_len):
            tok_sparse_idx = sparse_indices[global_token_idx]
            valid_mask = tok_sparse_idx >= 0
            valid_indices = tok_sparse_idx[valid_mask].long()

            q_tok = q_mqa[q_idx : q_idx + 1]  # [1, num_heads, head_dim]
            k_sparse = k_mqa[valid_indices]  # [num_valid, head_dim]
            v_sparse = v_mqa[valid_indices]  # [num_valid, kv_lora_rank]

            k_sparse = k_sparse.unsqueeze(1).expand(-1, num_heads, -1)
            v_sparse = v_sparse.unsqueeze(1).expand(-1, num_heads, -1)

            # SDPA: [1, num_heads, 1, head_dim] x [1, num_heads, num_valid, head_dim]
            q_sdpa_in = q_tok.unsqueeze(0).transpose(1, 2)
            k_sdpa_in = k_sparse.unsqueeze(0).transpose(1, 2)
            v_sdpa_in = v_sparse.unsqueeze(0).transpose(1, 2)

            sdpa_out = torch.nn.functional.scaled_dot_product_attention(
                q_sdpa_in, k_sdpa_in, v_sdpa_in, scale=scale
            )
            sdpa_out = sdpa_out.transpose(1, 2).squeeze(
                0
            )  # [1, num_heads, kv_lora_rank]

            sdpa_out = torch.einsum("qnl,lnv->qnv", sdpa_out, W_UV)
            reference_outputs.append(sdpa_out.flatten(start_dim=-2))

            global_token_idx += 1

        all_q_vllm.append(q_c)
        all_kv_c_vllm.append(kv_c_full[ctx_len:])
        all_k_pe_vllm.append(k_pe_full[ctx_len:])
        kv_c_contexts.append(kv_c_full[: ctx_len + 1])
        k_pe_contexts.append(k_pe_full[: ctx_len + 1])

    query_vllm = torch.cat(all_q_vllm, dim=0)
    kv_c_vllm = torch.cat(all_kv_c_vllm, dim=0)
    k_pe_vllm = torch.cat(all_k_pe_vllm, dim=0)
    sdpa_reference = torch.cat(reference_outputs, dim=0)

    vllm_config.cache_config.cache_dtype = kv_cache_dtype
    vllm_config.model_config.hf_text_config.index_topk = topk_tokens

    common_attn_metadata = create_common_attn_metadata(
        batch_spec,
        vllm_config.cache_config.block_size,
        device,
        arange_block_indices=True,
    )

    kv_cache = create_and_prepopulate_kv_cache(
        kv_c_contexts=kv_c_contexts,
        k_pe_contexts=k_pe_contexts,
        block_size=vllm_config.cache_config.block_size,
        head_size=head_size,
        dtype=dtype,
        device=device,
        num_blocks=vllm_config.cache_config.num_gpu_blocks,
        common_attn_metadata=common_attn_metadata,
        randomize_blocks=False,
        kv_cache_dtype=kv_cache_dtype,
        scale=kv_cache_scale,
    )
    if block_stride_rows is not None:
        pad = (0, 0, 0, block_stride_rows - block_size)
        kv_cache = torch.nn.functional.pad(kv_cache, pad)[:, :, :block_size]
        slot = int(common_attn_metadata.slot_mapping[-1])
        query_row = kv_cache[slot // block_size, 0, slot % block_size].zero_()

    # The sparse builder clones the layer's dense-MHA prefill backend from
    # static_forward_context; register a mock layer carrying one.
    prefill_backend = get_mla_prefill_backend(vllm_config)(
        num_heads=num_heads,
        scale=scale,
        kv_lora_rank=kv_lora_rank,
        qk_nope_head_dim=qk_nope_head_dim,
        qk_rope_head_dim=qk_rope_head_dim,
        v_head_dim=v_head_dim,
        vllm_config=vllm_config,
    )
    vllm_config.compilation_config.static_forward_context["placeholder"] = (
        SimpleNamespace(prefill_backend=prefill_backend, impl=SimpleNamespace())
    )

    builder_cls = backend_cls.get_builder_cls()
    builder = builder_cls(kv_cache_spec, ["placeholder"], vllm_config, device)
    metadata = builder.build(
        common_prefix_len=0, common_attn_metadata=common_attn_metadata
    )

    # Use the pre-computed sparse_indices for the mock indexer
    mock_indexer = SimpleNamespace(topk_indices_buffer=sparse_indices)

    kv_b_proj_weight = torch.cat([W_UK, W_UV], dim=-1)
    kv_b_proj_weight = kv_b_proj_weight.view(
        kv_lora_rank, num_heads * (qk_nope_head_dim + v_head_dim)
    )

    mock_kv_b_proj = ColumnParallelLinear(
        input_size=kv_lora_rank,
        output_size=num_heads * (qk_nope_head_dim + v_head_dim),
        bias=False,
    ).to(device=device, dtype=dtype)
    mock_kv_b_proj.weight = torch.nn.Parameter(kv_b_proj_weight.T.contiguous())

    impl_cls = backend_cls.get_impl_cls()
    with set_current_vllm_config(vllm_config):
        impl = impl_cls(
            num_heads=num_heads,
            head_size=head_size,
            scale=scale,
            num_kv_heads=1,
            alibi_slopes=None,
            sliding_window=None,
            kv_cache_dtype=vllm_config.cache_config.cache_dtype,
            logits_soft_cap=None,
            attn_type="decoder",
            kv_sharing_target_layer_name=None,
            q_lora_rank=None,
            kv_lora_rank=kv_lora_rank,
            qk_nope_head_dim=qk_nope_head_dim,
            qk_rope_head_dim=qk_rope_head_dim,
            qk_head_dim=qk_nope_head_dim + qk_rope_head_dim,
            v_head_dim=v_head_dim,
            kv_b_proj=mock_kv_b_proj,
            indexer=mock_indexer,
        )

        impl.process_weights_after_loading(dtype)

        # Create mock sparse MLA layer with weight matrices
        mock_layer = MockSparseMLAAttentionLayer(
            impl=impl,
            num_heads=num_heads,
            qk_nope_head_dim=qk_nope_head_dim,
            qk_rope_head_dim=qk_rope_head_dim,
            v_head_dim=v_head_dim,
            kv_lora_rank=kv_lora_rank,
            device=device,
            W_UK=W_UK,
            W_UV=W_UV,
            q_scale=q_scale,
            k_scale=k_scale,
        )

    out_buffer = torch.empty(
        metadata.num_actual_tokens, num_heads * v_head_dim, dtype=dtype, device=device
    )

    with torch.inference_mode():
        if backend_cls == FlashMLASparseBackend and kv_cache_dtype == "fp8_ds_mla":
            from vllm.v1.worker.workspace import current_workspace_manager

            current_workspace_manager().lock()
        backend_output = mock_layer.forward_impl(
            query_vllm,
            kv_c_vllm,
            k_pe_vllm,
            kv_cache,
            metadata,
            out_buffer,
        )

    if block_stride_rows is not None and kv_cache_dtype == "auto":
        expected_row = torch.cat((kv_c_vllm[-1], k_pe_vllm[-1, 0]))
        torch.testing.assert_close(query_row, expected_row, rtol=0, atol=0)
    elif block_stride_rows is not None:
        assert query_row.any()

    assert backend_output.shape == sdpa_reference.shape
    assert backend_output.dtype == sdpa_reference.dtype
    assert torch.isfinite(backend_output).all()

    # FP8 quantization introduces some error, but should be within reasonable bounds
    # BF16 (auto) should be very accurate, FP8 allows slightly more tolerance
    if kv_cache_dtype.startswith("fp8"):
        torch.testing.assert_close(
            backend_output, sdpa_reference, rtol=0.065, atol=0.05
        )
    else:
        torch.testing.assert_close(backend_output, sdpa_reference, rtol=0.01, atol=0.01)


@pytest.mark.parametrize(
    ("backend_cls", "kv_cache_dtype", "block_size", "block_stride_rows", "head_dims"),
    [
        pytest.param(
            FlashInferMLASparseTRTLLMBackend,
            "auto",
            256,
            320,
            (128, 128, 64, 128),
            id="FlashInferTRTLLM",
        ),
        # GLM-5.3 on SM90: the fp8 kernel's 64-token pages in larger blocks.
        pytest.param(
            FlashMLASparseBackend,
            "fp8_ds_mla",
            128,
            192,
            (64, 256, 0, 256),
            id="FlashMLA-fp8_ds_mla",
        ),
    ],
)
def test_sparse_mla_packed_stride(
    default_vllm_config,
    dist_init,
    workspace_init,
    monkeypatch,
    backend_cls,
    kv_cache_dtype,
    block_size,
    block_stride_rows,
    head_dims,
):
    test_sparse_backend_decode_correctness(
        default_vllm_config,
        dist_init,
        backend_cls,
        "mixed_small",
        kv_cache_dtype,
        1,
        block_size,
        workspace_init,
        1.0,
        1.0,
        *head_dims,
        monkeypatch,
        block_stride_rows=block_stride_rows,
    )


def _triton_convert_reference_impl(
    req_ids: torch.Tensor,
    block_table: torch.Tensor,
    token_indices: torch.Tensor,
    block_size: int,
    num_topk_tokens: int,
    HAS_PREFILL_WORKSPACE: bool = False,
    prefill_workspace_request_ids: torch.Tensor | None = None,
    prefill_workspace_starts: torch.Tensor | None = None,
) -> torch.Tensor:
    """Reference implementation for triton_convert_req_index_to_global_index."""
    num_tokens = req_ids.shape[0]
    max_blocks_per_req = block_table.shape[1]
    result = torch.empty(
        num_tokens, num_topk_tokens, dtype=torch.int32, device=req_ids.device
    )

    for token_id in range(num_tokens):
        req_id = req_ids[token_id].item()

        # Determine if this token uses workspace or paged cache
        use_prefill_workspace = False
        workspace_start = 0
        if HAS_PREFILL_WORKSPACE and prefill_workspace_request_ids is not None:
            assert prefill_workspace_starts is not None
            prefill_req_id = prefill_workspace_request_ids[token_id].item()
            if prefill_req_id >= 0:
                use_prefill_workspace = True
                workspace_start = prefill_workspace_starts[prefill_req_id].item()

        for idx_id in range(num_topk_tokens):
            token_idx = token_indices[token_id, idx_id].item()

            if token_idx == -1:
                result[token_id, idx_id] = -1
            elif use_prefill_workspace:
                # Prefill + using prefill workspace: map to workspace offset
                result[token_id, idx_id] = workspace_start + token_idx
            else:
                # Decode: map to paged cache
                block_id = token_idx // block_size
                if block_id >= max_blocks_per_req:
                    result[token_id, idx_id] = -1
                else:
                    block_num = block_table[req_id, block_id].item()
                    offset = token_idx % block_size
                    result[token_id, idx_id] = block_num * block_size + offset

    return result


@pytest.mark.parametrize("block_size", [16, 64, 128])
@pytest.mark.parametrize("num_topk_tokens", [128, 256, 512])
@pytest.mark.skipif(
    torch.cuda.get_device_capability() < (9, 0),
    reason="FlashMLASparseBackend requires CUDA 9.0 or higher",
)
def test_triton_convert_req_index_to_global_index_decode_only(
    block_size, num_topk_tokens
):
    device = torch.device(DEVICE_TYPE)
    num_tokens = 8
    num_requests = 4
    max_blocks_per_req = 10

    req_id = torch.randint(
        0, num_requests, (num_tokens,), dtype=torch.int32, device=device
    )
    block_table = torch.randint(
        0, 100, (num_requests, max_blocks_per_req), dtype=torch.int32, device=device
    )

    token_indices = torch.randint(
        0,
        block_size * max_blocks_per_req,
        (num_tokens, num_topk_tokens),
        dtype=torch.int32,
        device=device,
    )

    # Set some to -1 to test masking
    token_indices[0, :10] = -1
    token_indices[3, 50:60] = -1

    # Set some to out of bounds
    token_indices[2, 100:110] = max_blocks_per_req * block_size
    token_indices[6, 150:160] = max_blocks_per_req * block_size

    result = triton_convert_req_index_to_global_index(
        req_id,
        block_table,
        token_indices,
        BLOCK_SIZE=block_size,
        NUM_TOPK_TOKENS=num_topk_tokens,
    )

    reference_result = _triton_convert_reference_impl(
        req_id,
        block_table,
        token_indices,
        block_size,
        num_topk_tokens,
    )

    torch.testing.assert_close(result, reference_result, rtol=0, atol=0)


def test_index_group_convert_during_piecewise_capture():
    """In piecewise cudagraph mode the indexer is captured while the convert
    runs in the following eager break. The convert's side stream must not
    wait on ``logical_topk_ready`` there: an event recorded inside a captured
    segment is graph-local, and an eager wait on it raises
    cudaErrorInvalidValue."""
    device = torch.device(DEVICE_TYPE)
    num_tokens, num_topk, num_requests, blocks_per_req, block_size = 8, 128, 4, 4, 16

    logical_topk_indices = torch.randint(
        0,
        block_size * blocks_per_req,
        (num_tokens, num_topk),
        dtype=torch.int32,
        device=device,
    )
    builder = SparseMLAIndexGroupBuilder(logical_topk_indices)
    group, layer_index = builder.register_layer(is_index_producing_layer=True)
    assert layer_index == 0 and group.has_indexer

    req_id = torch.arange(num_tokens, dtype=torch.int32, device=device) % num_requests
    block_table = torch.arange(
        num_requests * blocks_per_req, dtype=torch.int32, device=device
    ).view(num_requests, blocks_per_req)
    attn_metadata = SimpleNamespace(
        req_id_per_token=req_id, block_table=block_table, block_size=block_size
    )
    expected = triton_convert_req_index_to_global_index(
        req_id,
        block_table,
        logical_topk_indices,
        BLOCK_SIZE=block_size,
        NUM_TOPK_TOKENS=num_topk,
    )

    vllm_config = create_vllm_config(model_name="Qwen/Qwen3.5-0.8B")
    marker = torch.zeros(1, device=device)
    capture_stream = torch.cuda.Stream(device=device)
    # torch.cuda.stream routes through vLLM's patched torch.cuda.set_stream,
    # so current_stream() tracks the capture stream as in production.
    with (
        torch.cuda.stream(capture_stream),
        set_forward_context(
            None, vllm_config, cudagraph_runtime_mode=CUDAGraphMode.PIECEWISE
        ),
    ):
        capture = BreakableCUDAGraphCapture()
        with capture:
            marker.add_(1)
            # Record inside a captured segment, convert in the eager
            # break — mirrors mla.py -> unified_mla_attention_with_output.
            group.set_logical_topk_ready(layer_index)
            capture.add_eager(
                lambda: group.convert_logical_to_physical_topk(
                    layer_index,
                    logical_topk_indices,
                    attn_metadata,
                    block_stride_rows=None,
                    return_valid_counts=False,
                )
            )
            marker.add_(1)
        capture.replay()
        capture_stream.synchronize()

    result = group.physical_topk_indices[:num_tokens]
    torch.testing.assert_close(result, expected, rtol=0, atol=0)


@pytest.mark.parametrize("block_size", [16])
@pytest.mark.skipif(
    torch.cuda.get_device_capability() < (9, 0),
    reason="FlashMLASparseBackend requires CUDA 9.0 or higher",
)
def test_triton_convert_req_index_to_global_index_with_prefill_workspace(block_size):
    device = torch.device(DEVICE_TYPE)
    num_requests = 4
    max_blocks_per_req = 8
    num_topk_tokens = 128

    # First 6 tokens are decode (reqs 0, 1), last 6 are prefill (reqs 2, 3)
    req_id = torch.tensor(
        [0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3], dtype=torch.int32, device=device
    )
    prefill_workspace_request_ids = torch.tensor(
        [-1, -1, -1, -1, -1, -1, 0, 0, 0, 1, 1, 1], dtype=torch.int32, device=device
    )

    # Workspace starts for the 2 prefill reqs: req 2 starts at 0, req 3 starts at 100
    prefill_workspace_starts = torch.tensor([0, 100], dtype=torch.int32, device=device)

    block_table = torch.randint(
        0, 50, (num_requests, max_blocks_per_req), dtype=torch.int32, device=device
    )
    token_indices = torch.randint(
        0,
        block_size * max_blocks_per_req,
        (req_id.shape[0], num_topk_tokens),
        dtype=torch.int32,
        device=device,
    )

    # Set some to -1 to test masking
    token_indices[0, :10] = -1
    token_indices[3, 50:60] = -1

    # Set some to out of bounds
    token_indices[2, 100:110] = max_blocks_per_req * block_size
    token_indices[6, 150:160] = max_blocks_per_req * block_size

    result = triton_convert_req_index_to_global_index(
        req_id,
        block_table,
        token_indices,
        BLOCK_SIZE=block_size,
        NUM_TOPK_TOKENS=num_topk_tokens,
        HAS_PREFILL_WORKSPACE=True,
        prefill_workspace_request_ids=prefill_workspace_request_ids,
        prefill_workspace_starts=prefill_workspace_starts,
    )

    reference_result = _triton_convert_reference_impl(
        req_id,
        block_table,
        token_indices,
        block_size,
        num_topk_tokens,
        HAS_PREFILL_WORKSPACE=True,
        prefill_workspace_request_ids=prefill_workspace_request_ids,
        prefill_workspace_starts=prefill_workspace_starts,
    )

    torch.testing.assert_close(result, reference_result, rtol=0, atol=0)


@pytest.mark.skipif(
    torch.cuda.get_device_capability() < (9, 0),
    reason="FlashMLASparseBackend requires CUDA 9.0 or higher",
)
def test_triton_convert_rejects_req_id_longer_than_token_indices():
    """Guard against the #47327 regression: the kernel grid is sized by
    req_id but the output is allocated like token_indices, so a full-batch
    req_id combined with an MQA-subset token_indices wrote past the end of
    the output buffer. The wrapper must reject the length mismatch instead
    of corrupting memory."""
    device = torch.device(DEVICE_TYPE)
    num_topk_tokens = 128
    block_size = 64
    block_table = torch.arange(40, dtype=torch.int32, device=device).view(4, 10)

    # Full batch: 2 decode tokens + 10 prefill tokens
    req_id_full = torch.tensor(
        [0, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3], dtype=torch.int32, device=device
    )
    num_mqa_tokens = 2
    token_indices = torch.randint(
        0,
        block_size * 10,
        (num_mqa_tokens, num_topk_tokens),
        dtype=torch.int32,
        device=device,
    )

    with pytest.raises(AssertionError, match="must cover the same tokens"):
        triton_convert_req_index_to_global_index(
            req_id_full,
            block_table,
            token_indices,
            BLOCK_SIZE=block_size,
            NUM_TOPK_TOKENS=num_topk_tokens,
        )

    # The sliced call is the intended usage and must match the reference.
    result = triton_convert_req_index_to_global_index(
        req_id_full[:num_mqa_tokens],
        block_table,
        token_indices,
        BLOCK_SIZE=block_size,
        NUM_TOPK_TOKENS=num_topk_tokens,
    )
    reference = _triton_convert_reference_impl(
        req_id_full[:num_mqa_tokens],
        block_table,
        token_indices,
        block_size,
        num_topk_tokens,
    )
    torch.testing.assert_close(result, reference, rtol=0, atol=0)


@pytest.mark.skipif(
    torch.cuda.get_device_capability() < (9, 0),
    reason="FlashMLASparseBackend requires CUDA 9.0 or higher",
)
def test_flashmla_forward_bf16_kv_slices_req_id_to_mqa_tokens():
    """Guard against the #47327 regression: when the dense-MHA prefill split
    is active, forward_mqa only receives the leading decode tokens, but
    _forward_bf16_kv passed the full-batch req_id_per_token to the index
    conversion, making it write past the end of its output buffer. The call
    site must slice req_id_per_token to the MQA tokens."""
    device = torch.device(DEVICE_TYPE)
    num_topk_tokens = 128
    block_size = 64
    num_batch_tokens = 12
    num_mqa_tokens = 2

    attn_metadata = SimpleNamespace(
        req_id_per_token=torch.tensor(
            [0, 1] + [2] * 5 + [3] * 5, dtype=torch.int32, device=device
        ),
        block_table=torch.arange(40, dtype=torch.int32, device=device).view(4, 10),
        block_size=block_size,
        num_decode_tokens=num_mqa_tokens,
        num_actual_tokens=num_mqa_tokens,
    )
    assert attn_metadata.req_id_per_token.shape[0] == num_batch_tokens

    q = torch.zeros(num_mqa_tokens, 4, 576, dtype=torch.bfloat16, device=device)
    kv_cache = torch.zeros(40, block_size, 576, dtype=torch.bfloat16, device=device)
    topk_indices = torch.randint(
        0,
        block_size * 10,
        (num_mqa_tokens, num_topk_tokens),
        dtype=torch.int32,
        device=device,
    )

    captured = {}

    def _stub_kernel(q, kv, indices, lengths, actual_num_heads):
        captured["indices"] = indices
        captured["actual_num_heads"] = actual_num_heads
        return (
            torch.zeros(q.shape[0], q.shape[1], 512, dtype=q.dtype, device=q.device),
            None,
        )

    def _convert_topk(indices, metadata, **kwargs):
        return triton_convert_req_index_to_global_index(
            metadata.req_id_per_token[: indices.shape[0]],
            metadata.block_table,
            indices,
            BLOCK_SIZE=metadata.block_size,
            BLOCK_STRIDE_ROWS=kwargs["block_stride_rows"],
            NUM_TOPK_TOKENS=indices.shape[1],
            return_valid_counts=kwargs["return_valid_counts"],
        )

    stub_impl = SimpleNamespace(
        _bf16_flash_mla_kernel=_stub_kernel,
        _convert_logical_to_physical_topk=_convert_topk,
        index_group=None,
        index_group_index=0,
        pcp_dcp_kv_gather=False,
    )

    out, _ = FlashMLASparseImpl._forward_bf16_kv(
        stub_impl, q, kv_cache, topk_indices, attn_metadata, q.shape[1]
    )

    assert out.shape[0] == num_mqa_tokens
    assert captured["indices"].shape[0] == num_mqa_tokens
    assert captured["actual_num_heads"] == q.shape[1]
    reference = _triton_convert_reference_impl(
        attn_metadata.req_id_per_token[:num_mqa_tokens],
        attn_metadata.block_table,
        topk_indices,
        block_size,
        num_topk_tokens,
    )
    torch.testing.assert_close(captured["indices"], reference, rtol=0, atol=0)


def test_flashmla_hisparse_resident_mixed_batch_skips_host_staging():
    """A resident prefill batched with decodes reads resident KV in one kernel
    instead of staging its whole history from host."""
    num_tokens = 6
    q = torch.zeros(num_tokens, 4, 576, dtype=torch.bfloat16, device=DEVICE_TYPE)
    topk = torch.zeros(num_tokens, 4, dtype=torch.int32, device=DEVICE_TYPE)
    counts = torch.full((num_tokens,), 4, dtype=torch.int32, device=DEVICE_TYPE)
    kernel_rows: list[int] = []

    def kernel(q, kv, indices, lengths, actual_num_heads):  # noqa: ARG001
        kernel_rows.append(q.shape[0])
        return q[..., :512], None

    index_group = object.__new__(HiSparseMLAIndexGroup)
    index_group.caches = [SimpleNamespace(all_context_pages_resident=True)]
    index_group.convert_logical_to_physical_topk = MagicMock(
        return_value=(topk, counts)
    )
    index_group.physical_kv_cache = MagicMock(
        return_value=torch.empty(1, device=DEVICE_TYPE)
    )
    index_group.stage_prefill_rows = MagicMock(side_effect=AssertionError)
    impl = SimpleNamespace(
        _bf16_flash_mla_kernel=kernel,
        index_group=index_group,
        index_group_index=0,
    )
    metadata = SimpleNamespace(
        req_id_per_token=torch.zeros(num_tokens, dtype=torch.int32),
        block_table=torch.zeros((2, 1), dtype=torch.int32),
        num_decode_tokens=2,
    )

    out, _ = FlashMLASparseImpl._forward_bf16_kv(
        impl, q, torch.empty(1, device=DEVICE_TYPE), topk, metadata, q.shape[1]
    )

    assert kernel_rows == [num_tokens]
    assert out.shape == (num_tokens, 4, 512)


@pytest.mark.parametrize(
    "seq_lens,max_buf,expected",
    [
        # Basic split: totals per chunk ≤ max_buf
        (torch.tensor([2, 3, 4, 2]), 5, [(0, 2), (2, 3), (3, 4)]),
        # Exact fits should split between items when adding the next would overflow
        (torch.tensor([5, 5, 5]), 5, [(0, 1), (1, 2), (2, 3)]),
        # All requests fit in a single chunk
        (torch.tensor([1, 1, 1]), 10, [(0, 3)]),
        # Large buffer
        (torch.tensor([4, 4, 4]), 100, [(0, 3)]),
    ],
)
def test_split_prefill_chunks(seq_lens, max_buf, expected):
    out = split_prefill_chunks(seq_lens, max_buf)
    assert out == expected


@pytest.mark.parametrize(
    ("max_query_len", "expected"),
    [(32768, True), (33024, False)],
)
def test_masked_mha_workspace_fits_single_request_boundary(max_query_len, expected):
    """A 32K prefill needs the default workspace exactly; shrinking it would
    push a supported request onto MQA."""
    assert (
        _masked_mha_workspace_fits(
            batch_size=1,
            max_query_len=max_query_len,
            max_context_chunk_seq_len=0,
            workspace_numel=GLOBAL_TOPK_MASK_MAX_BYTES // torch.int32.itemsize,
        )
        is expected
    )


@pytest.mark.parametrize(
    ("backend_name", "tensor_parallel_size", "query_len"),
    [
        ("FLASHMLA_SPARSE", 4, 48 * 1024),
        ("FLASHMLA_SPARSE", 8, 112 * 1024),
        ("FLASHINFER_MLA_SPARSE", 4, 36 * 1024),
        ("FLASHINFER_MLA_SPARSE", 8, 64 * 1024),
        ("FLASH_ATTN_MLA_SPARSE_FA4", 4, 36 * 1024),
    ],
)
def test_masked_mha_workspace_guards_long_routing_policy(
    backend_name, tensor_parallel_size, query_len
):
    assert _use_masked_mha(
        backend_name=backend_name,
        tensor_parallel_size=tensor_parallel_size,
        qk_head_dim=256,
        v_head_dim=256,
        query_len=query_len,
        seq_len=query_len,
        has_context=False,
    )
    assert not _masked_mha_workspace_fits(
        batch_size=1,
        max_query_len=query_len,
        max_context_chunk_seq_len=0,
        workspace_numel=GLOBAL_TOPK_MASK_MAX_BYTES // torch.int32.itemsize,
    )


@pytest.mark.parametrize(
    ("model_dims", "kv_cache_dtype", "fa_version", "expected"),
    [
        pytest.param((128, 512, 128, 64, 128), "auto", 4, True, id="deepseek_v32"),
        pytest.param((64, 512, 192, 64, 256), "auto", 4, True, id="glm5"),
        pytest.param((64, 512, 256, 0, 256), "auto", 4, True, id="glm53_flash_nope"),
        pytest.param((64, 512, 256, 0, 256), "fp8", 4, False, id="quantized_kv"),
        pytest.param((64, 512, 256, 0, 256), "auto", 3, False, id="no_fa4"),
        pytest.param((64, 512, 256, 64, 256), "auto", 4, False, id="rope_320"),
        pytest.param((128, 512, 256, 0, 256), "auto", 4, False, id="wrong_heads"),
    ],
)
def test_is_masked_mha_available_model_dims(
    monkeypatch, model_dims, kv_cache_dtype, fa_version, expected
):
    """The allow-list gates masked MHA per exact model geometry: the DeepSeek-V3.2,
    GLM-5 and NoPE GLM-5.3-Flash layouts on an SM100-family GPU with FA4 and an
    unquantized KV cache, nothing else."""
    import vllm.model_executor.layers.attention.sparse_mla_attention as mod

    monkeypatch.setattr(
        mod.current_platform, "is_device_capability_family", lambda family: True
    )
    monkeypatch.setattr(mod, "get_flash_attn_version", lambda **kwargs: fa_version)
    assert _is_masked_mha_available(*model_dims, kv_cache_dtype) is expected


def test_masked_mha_workspace_fits_accounts_for_batch_and_context():
    """Request count and context chunk length are independent multipliers."""
    base = dict(batch_size=2, max_query_len=2048, max_context_chunk_seq_len=2048)
    exact = math.prod(_topk_mask_shape(2, 2048, 2048))

    assert _masked_mha_workspace_fits(**base, workspace_numel=exact)
    assert not _masked_mha_workspace_fits(
        **{**base, "batch_size": 3}, workspace_numel=exact
    )
    assert not _masked_mha_workspace_fits(
        **{**base, "max_context_chunk_seq_len": 4096}, workspace_numel=exact
    )


PREFILL_BATCH_SPECS = {
    "short_dense_mha": BatchSpec(seq_lens=[64, 128], query_lens=[64, 128]),
    "short_context_dense_mha": BatchSpec(seq_lens=[128, 160], query_lens=[64, 32]),
    "masked_mha": BatchSpec(seq_lens=[256], query_lens=[256]),
    "masked_mha_chunked_context": BatchSpec(seq_lens=[448, 384], query_lens=[256, 256]),
}


@pytest.mark.skipif(
    current_platform.is_cuda() and torch.cuda.get_device_capability()[0] < 10,
    reason="Sparse MLA forward_mha requires FA4 (SM100+)",
)
@pytest.mark.parametrize("batch_name", list(PREFILL_BATCH_SPECS.keys()))
@pytest.mark.parametrize(
    "kv_cache_dtype", (["auto", "fp8"] if current_platform.is_rocm() else ["auto"])
)
@pytest.mark.parametrize(
    ("num_heads", "qk_nope_head_dim", "qk_rope_head_dim", "v_head_dim"),
    [
        pytest.param(128, 128, 64, 128, id="deepseek_hd192_v128"),
        pytest.param(64, 192, 64, 256, id="glm5_hd256_v256"),
        pytest.param(64, 256, 0, 256, id="glm53_flash_nope_hd256_v256"),
    ],
)
def test_sparse_backend_prefill_correctness(
    default_vllm_config,
    dist_init,
    batch_name,
    kv_cache_dtype,
    num_heads,
    qk_nope_head_dim,
    qk_rope_head_dim,
    v_head_dim,
    workspace_init,
):
    """Test the normal dense forward_mha path for sparse MLA short prefill."""
    backend_cls = (
        ROCMAiterMLASparseBackend
        if current_platform.is_rocm()
        else FlashMLASparseBackend
    )
    if current_platform.is_rocm() and (
        batch_name.startswith("masked_mha") or qk_nope_head_dim != 128
    ):
        pytest.skip("ROCm dense prefill requires the DeepSeek MLA layout")
    batch_spec = PREFILL_BATCH_SPECS[batch_name]

    device = torch.device(DEVICE_TYPE)
    dtype = torch.bfloat16
    block_size = 64

    kv_lora_rank = 512
    head_size = kv_lora_rank + qk_rope_head_dim
    masked_mha = batch_name.startswith("masked_mha")
    topk_tokens = 200 if masked_mha else 512

    max_seqlen = max(batch_spec.seq_lens)
    total_cache_tokens = sum(batch_spec.seq_lens)

    vllm_config = create_vllm_config(
        model_name="deepseek-ai/DeepSeek-V2-Lite-Chat",
        tensor_parallel_size=1,
        max_model_len=max_seqlen,
        num_gpu_blocks=max(2048, cdiv(total_cache_tokens, block_size) + 1),
        block_size=block_size,
        hf_config_override={
            "index_topk": topk_tokens,
            "attn_module_list_cfg": [{"topk_tokens": topk_tokens}],
        },
    )
    model_config = vllm_config.model_config
    model_config.hf_text_config = SimpleNamespace(
        index_topk=topk_tokens,
        q_lora_rank=None,
        kv_lora_rank=kv_lora_rank,
        qk_nope_head_dim=qk_nope_head_dim,
        qk_rope_head_dim=qk_rope_head_dim,
        v_head_dim=v_head_dim,
        model_type="deepseek_v2",
    )
    del model_config.hf_config.index_topk  # Composite configs only nest this field.
    model_config.dtype = dtype
    model_config.model_arch_config.total_num_attention_heads = num_heads
    model_config.get_num_attention_heads = MethodType(
        lambda self, parallel_config: num_heads, model_config
    )
    model_config.get_num_kv_heads = MethodType(
        lambda self, parallel_config: 1, model_config
    )
    model_config.get_head_size = MethodType(lambda self: head_size, model_config)
    model_config.get_sliding_window = MethodType(lambda self: None, model_config)

    kv_cache_spec = create_standard_kv_cache_spec(vllm_config)
    scale = 1.0 / math.sqrt(qk_nope_head_dim + qk_rope_head_dim)

    torch.manual_seed(42)

    W_UK = torch.rand(
        kv_lora_rank, num_heads, qk_nope_head_dim, dtype=dtype, device=device
    )
    W_UV = torch.rand(kv_lora_rank, num_heads, v_head_dim, dtype=dtype, device=device)

    seq_lens = batch_spec.seq_lens
    query_lens = batch_spec.query_lens

    # Compute dense reference outputs.
    total_query_tokens = sum(query_lens)
    sparse_indices = torch.full(
        (total_query_tokens, topk_tokens), -1, dtype=torch.int32, device=device
    )

    all_q, all_kv_c_new, all_k_pe_new = [], [], []
    kv_c_contexts, k_pe_contexts = [], []
    reference_outputs = []
    global_token_idx = 0

    for i in range(batch_spec.batch_size):
        s_len = seq_lens[i]
        q_len = query_lens[i]
        ctx_len = s_len - q_len

        q_mha = torch.rand(
            q_len,
            num_heads,
            qk_nope_head_dim + qk_rope_head_dim,
            dtype=dtype,
            device=device,
        )
        kv_c_full = torch.rand(s_len, kv_lora_rank, dtype=dtype, device=device)
        k_pe_full = torch.rand(s_len, 1, qk_rope_head_dim, dtype=dtype, device=device)

        # Decompress all KV for reference
        kv_b_weight = torch.cat([W_UK, W_UV], dim=-1).view(
            kv_lora_rank, num_heads * (qk_nope_head_dim + v_head_dim)
        )
        kv_decompressed = (kv_c_full @ kv_b_weight).view(
            s_len, num_heads, qk_nope_head_dim + v_head_dim
        )
        k_nope_all, v_all = kv_decompressed.split(
            [qk_nope_head_dim, v_head_dim], dim=-1
        )
        k_pe_expanded = k_pe_full.expand(-1, num_heads, -1)
        k_all = torch.cat([k_nope_all, k_pe_expanded], dim=-1)

        for j in range(q_len):
            attend_end = ctx_len + j + 1
            q_tok = q_mha[j : j + 1]  # (1, H, D_qk)
            if masked_mha:
                actual_topk = min(topk_tokens, attend_end)
                attend_indices = torch.randperm(attend_end, device=device)[:actual_topk]
                sparse_indices[global_token_idx, :actual_topk] = attend_indices
                k_attend = k_all[attend_indices]
                v_attend = v_all[attend_indices]
            else:
                k_attend = k_all[:attend_end]  # (N, H, D_qk)
                v_attend = v_all[:attend_end]  # (N, H, D_v)

            q_sdpa = q_tok.unsqueeze(0).transpose(1, 2).float()
            k_sdpa = k_attend.unsqueeze(0).transpose(1, 2).float()
            v_sdpa = v_attend.unsqueeze(0).transpose(1, 2).float()

            out = torch.nn.functional.scaled_dot_product_attention(
                q_sdpa, k_sdpa, v_sdpa, scale=scale
            )
            out = out.transpose(1, 2).squeeze(0)  # (1, H, D_v)
            reference_outputs.append(out.to(dtype).flatten(start_dim=-2))
            global_token_idx += 1

        all_q.append(q_mha)
        all_kv_c_new.append(kv_c_full[ctx_len:])
        all_k_pe_new.append(k_pe_full[ctx_len:])
        kv_c_contexts.append(kv_c_full)
        k_pe_contexts.append(k_pe_full)

    query_cat = torch.cat(all_q, dim=0)
    kv_c_cat = torch.cat(all_kv_c_new, dim=0)
    k_pe_cat = torch.cat(all_k_pe_new, dim=0)
    ref_output = torch.cat(reference_outputs, dim=0)

    vllm_config.cache_config.cache_dtype = kv_cache_dtype
    vllm_config.model_config.hf_text_config.index_topk = topk_tokens

    common_attn_metadata = create_common_attn_metadata(
        batch_spec,
        vllm_config.cache_config.block_size,
        device,
        arange_block_indices=True,
    )

    kv_cache = create_and_prepopulate_kv_cache(
        kv_c_contexts=kv_c_contexts,
        k_pe_contexts=k_pe_contexts,
        block_size=block_size,
        head_size=head_size,
        dtype=dtype,
        device=device,
        num_blocks=vllm_config.cache_config.num_gpu_blocks,
        common_attn_metadata=common_attn_metadata,
        randomize_blocks=False,
        kv_cache_dtype=kv_cache_dtype,
    )

    # The sparse builder clones the layer's dense-MHA prefill backend from
    # static_forward_context; register a mock layer carrying one.
    prefill_backend = get_mla_prefill_backend(vllm_config)(
        num_heads=num_heads,
        scale=scale,
        kv_lora_rank=kv_lora_rank,
        qk_nope_head_dim=qk_nope_head_dim,
        qk_rope_head_dim=qk_rope_head_dim,
        v_head_dim=v_head_dim,
        vllm_config=vllm_config,
    )
    vllm_config.compilation_config.static_forward_context["placeholder"] = (
        SimpleNamespace(prefill_backend=prefill_backend, impl=SimpleNamespace())
    )

    builder_cls = backend_cls.get_builder_cls()
    builder = builder_cls(kv_cache_spec, ["placeholder"], vllm_config, device)
    if batch_name == "masked_mha_chunked_context":
        builder.chunked_prefill_workspace_size = block_size * batch_spec.batch_size
        builder.chunked_prefill_workspace = torch.empty(
            (builder.chunked_prefill_workspace_size, head_size),
            dtype=dtype,
            device=device,
        )
    # Drive the queries through the dense-MHA prefill path directly (the routing
    # threshold would otherwise classify these short queries as MQA decodes).
    builder.reorder_batch_threshold = 1
    metadata = builder.build(
        common_prefix_len=0, common_attn_metadata=common_attn_metadata
    )

    mock_indexer = SimpleNamespace(topk_indices_buffer=sparse_indices)

    kv_b_proj_weight = torch.cat([W_UK, W_UV], dim=-1).view(
        kv_lora_rank, num_heads * (qk_nope_head_dim + v_head_dim)
    )

    mock_kv_b_proj = ColumnParallelLinear(
        input_size=kv_lora_rank,
        output_size=num_heads * (qk_nope_head_dim + v_head_dim),
        bias=False,
    ).to(device=device, dtype=dtype)
    mock_kv_b_proj.weight = torch.nn.Parameter(kv_b_proj_weight.T.contiguous())

    impl_cls = backend_cls.get_impl_cls()
    with set_current_vllm_config(vllm_config):
        impl = impl_cls(
            num_heads=num_heads,
            head_size=head_size,
            scale=scale,
            num_kv_heads=1,
            alibi_slopes=None,
            sliding_window=None,
            kv_cache_dtype=kv_cache_dtype,
            logits_soft_cap=None,
            attn_type="decoder",
            kv_sharing_target_layer_name=None,
            q_lora_rank=None,
            kv_lora_rank=kv_lora_rank,
            qk_nope_head_dim=qk_nope_head_dim,
            qk_rope_head_dim=qk_rope_head_dim,
            qk_head_dim=qk_nope_head_dim + qk_rope_head_dim,
            v_head_dim=v_head_dim,
            kv_b_proj=mock_kv_b_proj,
            indexer=mock_indexer,
        )
        impl.process_weights_after_loading(dtype)

    out_buffer = torch.empty(
        total_query_tokens, num_heads * v_head_dim, dtype=dtype, device=device
    )

    with torch.inference_mode():
        impl.forward_mha(
            q=query_cat,
            kv_c_normed=kv_c_cat,
            k_pe=k_pe_cat,
            # Impls see the bind-time-squeezed [B, N, C] cache; mirror bind_kv_cache.
            kv_c_and_k_pe_cache=kv_cache.squeeze(1),
            attn_metadata=metadata,
            k_scale=torch.tensor(1.0, device=device),
            output=out_buffer,
        )

    assert out_buffer.shape == ref_output.shape
    assert torch.isfinite(out_buffer).all(), "Non-finite values in output"
    if kv_cache_dtype.startswith("fp8"):
        torch.testing.assert_close(out_buffer, ref_output, rtol=0.065, atol=0.05)
    else:
        torch.testing.assert_close(out_buffer, ref_output, rtol=0.01, atol=0.01)


@pytest.mark.parametrize(
    "seq_lens,query_lens,workspace_size,max_logits_bytes,expected",
    [
        (
            torch.tensor([0]),
            torch.tensor([0]),
            100,
            1000,
            [],
        ),
        # Logits constraint triggers split (M*N exceeds budget)
        # req0: M=10, N=100 -> 1000 elems (4000 bytes) - fits in 5000
        # req1: adding M=10, N=100 -> new_M=20, new_N=200 -> 4000 elems > 1250
        (
            torch.tensor([100, 100, 100]),
            torch.tensor([10, 10, 10]),
            1000,  # workspace allows all
            5000,  # 1250 float32 elems -> forces split
            [
                (slice(0, 1), slice(0, 10)),
                (slice(1, 2), slice(0, 10)),
                (slice(2, 3), slice(0, 10)),
            ],
        ),
        # Both constraints satisfied - all fit in one chunk
        (
            torch.tensor([10, 10, 10]),
            torch.tensor([5, 5, 5]),
            100,
            10000,  # 2500 elems, M*N = 15*30 = 450 < 2500
            [(slice(0, 3), slice(0, 15))],
        ),
        # Workspace constraint triggers first
        (
            torch.tensor([50, 50, 50]),
            torch.tensor([1, 1, 1]),
            50,  # workspace only fits one at a time
            1000000,  # logits budget is huge
            [
                (slice(0, 1), slice(0, 1)),
                (slice(1, 2), slice(0, 1)),
                (slice(2, 3), slice(0, 1)),
            ],
        ),
        # Greedy filling: first two fit, third doesn't
        # req0: M=5, N=10 -> 50 elems
        # req0+1: M=10, N=20 -> 200 elems <= 250
        # req0+1+2: M=15, N=30 -> 450 elems > 250
        (
            torch.tensor([10, 10, 10]),
            torch.tensor([5, 5, 5]),
            100,
            1000,  # 250 elems
            [(slice(0, 2), slice(0, 10)), (slice(2, 3), slice(0, 5))],
        ),
    ],
)
def test_split_indexer_prefill_chunks(
    seq_lens, query_lens, workspace_size, max_logits_bytes, expected
):
    out = DeepseekV32IndexerMetadataBuilder._split_indexer_prefill_chunks(
        seq_lens,
        query_lens,
        workspace_size,
        max_logits_bytes,
    )
    assert out == expected


def test_split_indexer_prefill_chunks_single_request_overflow():
    """Test that single request exceeding budget is sub-chunked on query dim."""
    seq_lens = torch.tensor([1000, 50])
    query_lens = torch.tensor([100, 5])

    out = DeepseekV32IndexerMetadataBuilder._split_indexer_prefill_chunks(
        seq_lens, query_lens, 2000, 1000
    )
    # max_logits_elems = 250, N=1000 -> max_q = 1 -> 100 query sub-chunks
    expected = [(slice(0, 1), slice(i, i + 1)) for i in range(100)]
    # req1: M=5, N=50 -> 250 elems fits budget
    expected.append((slice(1, 2), slice(0, 5)))
    assert out == expected


# Power-of-two, GLM's padded tile, and atomic fallback, with reused buffers.
@pytest.mark.parametrize(
    "num_topk_tokens,reuse_buffers",
    [(128, False), (2176, False), (2176, True), (4224, False), (4224, True)],
)
def test_triton_convert_returns_valid_counts(num_topk_tokens: int, reuse_buffers: bool):
    """Test that return_valid_counts correctly counts non-negative indices."""
    device = torch.device(DEVICE_TYPE)
    num_tokens = 8
    num_requests = 2
    max_blocks_per_req = 10
    block_size = 64

    req_id = torch.tensor([0, 0, 0, 0, 1, 1, 1, 1], dtype=torch.int32, device=device)
    block_table = torch.arange(
        num_requests * max_blocks_per_req, dtype=torch.int32, device=device
    ).view(num_requests, max_blocks_per_req)

    # Include partly valid, fully valid, and empty selections for each request.
    token_indices = torch.full(
        (num_tokens, num_topk_tokens), -1, dtype=torch.int32, device=device
    )
    valid_counts_per_token = [
        num_topk_tokens // 2,
        num_topk_tokens // 4,
        num_topk_tokens,
        0,
    ] * 2
    expected_valid = []
    for i in range(num_tokens):
        num_valid = valid_counts_per_token[i]
        token_indices[i, :num_valid] = torch.arange(
            num_valid, dtype=torch.int32, device=device
        ) % (block_size * max_blocks_per_req)
        expected_valid.append(num_valid)

    expected_valid_tensor = torch.tensor(
        expected_valid, dtype=torch.int32, device=device
    )

    buffers = {}
    if reuse_buffers:
        buffers = dict(
            out=torch.full_like(token_indices, -99),
            valid_counts_out=torch.full(
                (num_tokens,), 99, dtype=torch.int32, device=device
            ),
        )

    # Test with return_valid_counts=True
    result, valid_counts = triton_convert_req_index_to_global_index(
        req_id,
        block_table,
        token_indices,
        BLOCK_SIZE=block_size,
        NUM_TOPK_TOKENS=num_topk_tokens,
        return_valid_counts=True,
        **buffers,
    )

    torch.testing.assert_close(valid_counts, expected_valid_tensor, rtol=0, atol=0)

    # Test that return_valid_counts=False returns only the indices
    result_only = triton_convert_req_index_to_global_index(
        req_id,
        block_table,
        token_indices,
        BLOCK_SIZE=block_size,
        NUM_TOPK_TOKENS=num_topk_tokens,
        return_valid_counts=False,
    )
    assert isinstance(result_only, torch.Tensor)
    for row, num_valid in enumerate(expected_valid):
        compact_valid = result[row, :num_valid].sort().values
        original_valid = result_only[row][result_only[row] >= 0].sort().values
        torch.testing.assert_close(compact_valid, original_valid, rtol=0, atol=0)
        assert torch.all(result[row, num_valid:] == -1)


def test_sparse_mla_index_group_converts_decode_indices_once(monkeypatch):
    device = torch.device(DEVICE_TYPE)
    logical = torch.full((2, 128), -1, dtype=torch.int32, device=device)
    logical[:, :4] = torch.tensor(
        [[0, 2, 5, -1], [1, 3, 7, -1]], dtype=torch.int32, device=device
    )
    builder = SparseMLAIndexGroupBuilder(logical)
    group, leader_index = builder.register_layer(True)
    follower_group, follower_index = builder.register_layer(False)
    assert follower_group is group

    block_table = torch.tensor([[2, 3], [4, 5]], dtype=torch.int32, device=device)
    metadata = SimpleNamespace(
        block_table=block_table,
        block_size=4,
        req_id_per_token=torch.tensor([0, 1], dtype=torch.int32, device=device),
        num_decode_tokens=2,
        num_actual_tokens=2,
    )
    calls = []
    convert = index_group_module.triton_convert_req_index_to_global_index

    def convert_spy(*args, **kwargs):
        calls.append((args, kwargs))
        return convert(*args, **kwargs)

    monkeypatch.setattr(
        index_group_module,
        "triton_convert_req_index_to_global_index",
        convert_spy,
    )
    group.set_logical_topk_ready(leader_index)
    leader_result = group.convert_logical_to_physical_topk(
        leader_index,
        logical,
        metadata,
        block_stride_rows=None,
        return_valid_counts=True,
    )
    follower_result = group.convert_logical_to_physical_topk(
        follower_index,
        logical,
        metadata,
        block_stride_rows=None,
        return_valid_counts=True,
    )
    torch.accelerator.synchronize()

    assert len(calls) == 1
    leader_indices, leader_counts = leader_result
    follower_indices, follower_counts = follower_result
    torch.testing.assert_close(follower_indices, leader_indices)
    torch.testing.assert_close(follower_counts, leader_counts)


def test_sparse_mla_index_group_falls_back_for_prefill_sized_batch(monkeypatch):
    device = torch.device(DEVICE_TYPE)
    logical = torch.full((4, 128), -1, dtype=torch.int32, device=device)
    logical[:, 0] = torch.tensor([0, 1, 2, 3], dtype=torch.int32, device=device)
    builder = SparseMLAIndexGroupBuilder(logical, max_decode_rows=2)
    group, leader_index = builder.register_layer(True)
    follower_group, follower_index = builder.register_layer(False)
    assert follower_group is group

    metadata = SimpleNamespace(
        block_table=torch.tensor(
            [[2], [3], [4], [5]], dtype=torch.int32, device=device
        ),
        block_size=4,
        req_id_per_token=torch.arange(4, dtype=torch.int32, device=device),
    )
    calls = 0
    convert = index_group_module.triton_convert_req_index_to_global_index

    def convert_spy(*args, **kwargs):
        nonlocal calls
        calls += 1
        return convert(*args, **kwargs)

    monkeypatch.setattr(
        index_group_module,
        "triton_convert_req_index_to_global_index",
        convert_spy,
    )
    leader_result = group.convert_logical_to_physical_topk(
        leader_index,
        logical,
        metadata,
        block_stride_rows=None,
        return_valid_counts=False,
    )
    follower_result = group.convert_logical_to_physical_topk(
        follower_index,
        logical,
        metadata,
        block_stride_rows=None,
        return_valid_counts=False,
    )
    torch.accelerator.synchronize()

    assert calls == 2
    torch.testing.assert_close(follower_result, leader_result)


def test_sparse_mla_index_groups_own_distinct_physical_buffers():
    logical = torch.empty((8, 128), dtype=torch.int32, device=DEVICE_TYPE)
    builder = SparseMLAIndexGroupBuilder(logical, max_decode_rows=2)

    first, _ = builder.register_layer(True)
    follower, _ = builder.register_layer(False)
    second, _ = builder.register_layer(True)

    assert follower is first
    assert second is not first
    assert first.physical_topk_indices[:2].shape == (2, 128)
    assert second.physical_topk_indices[:2].shape == (2, 128)
    assert first.physical_topk_indices.data_ptr() != (
        second.physical_topk_indices.data_ptr()
    )


# HiSparse is host-resident-only and kernel-only: runtime construction
# raises without the compiled CUDA ops.
requires_hisparse_ops = pytest.mark.skipif(
    not _has_hisparse_ops(),
    reason="HiSparse CUDA ops not compiled",
)


def _make_hisparse_runtime(
    *,
    top_k: int = 4,
    device_buffer_size: int = 5,
    max_num_reqs: int = 2,
    row_width: int = 8,
    block_size: int = 64,
    max_swap_rows: int | None = None,
    index_group: hisparse_runtime.HiSparseIndexGroup | None = None,
    max_union_rows: int | None = None,
) -> HiSparseRuntime:
    runtime = HiSparseRuntime(
        config=ResolvedHiSparseConfig(
            top_k=top_k,
            device_buffer_size=device_buffer_size,
            max_union_rows=max_union_rows or device_buffer_size,
        ),
        max_num_reqs=max_num_reqs,
        row_width=row_width,
        kv_dtype=torch.float32,
        device=DEVICE_TYPE,
        max_swap_rows=max_swap_rows,
        index_group=index_group,
    )
    blocks_per_request = cdiv(runtime.region_stride, block_size)
    # Leave one extra physical block for resident-tier tests. HMA group layouts
    # alias this backing, but distinct group allocations use distinct block IDs.
    num_blocks = max_num_reqs * blocks_per_request + 1
    raw = torch.zeros(
        num_blocks * block_size * row_width,
        dtype=torch.float32,
        device=DEVICE_TYPE,
    ).view(torch.int8)
    block_table = torch.arange(
        max_num_reqs * blocks_per_request,
        dtype=torch.int32,
        device=DEVICE_TYPE,
    ).view(max_num_reqs, blocks_per_request)
    runtime.bind_hot_cache(
        raw,
        byte_offset=0,
        block_stride=block_size * row_width * torch.float32.itemsize,
        num_blocks=num_blocks,
        block_size=block_size,
        block_table=block_table,
    )
    runtime.request_state_indices = torch.arange(
        max_num_reqs, dtype=torch.int32, device=runtime.device
    )
    return runtime


def _make_hisparse_cache_handle(
    *,
    top_k: int = 4,
    device_buffer_size: int = 5,
    max_num_reqs: int = 2,
    row_width: int = 8,
    block_size: int = 64,
    max_swap_rows: int | None = None,
) -> HiSparseCacheHandle:
    runtime = _make_hisparse_runtime(
        top_k=top_k,
        device_buffer_size=device_buffer_size,
        max_num_reqs=max_num_reqs,
        row_width=row_width,
        block_size=block_size,
        max_swap_rows=max_swap_rows,
    )
    return HiSparseCacheHandle(runtime)


def _valid_rows(num_rows: int) -> torch.Tensor:
    return torch.tensor([num_rows], dtype=torch.int32, device=DEVICE_TYPE)


@requires_hisparse_ops
def test_hisparse_uses_graph_stable_request_state_mapping():
    device = torch.device(DEVICE_TYPE)
    block_size, row_width = 64, 8
    runtime = _make_hisparse_runtime(
        top_k=1,
        device_buffer_size=2,
        max_num_reqs=2,
        row_width=row_width,
        block_size=block_size,
    )
    runtime.bind_source_cache(
        torch.arange(2 * block_size * row_width, dtype=torch.float32)
        .view(2, block_size, row_width)
        .pin_memory()
    )
    runtime.request_state_indices = torch.tensor([1], dtype=torch.int32, device=device)

    cache_handle = HiSparseCacheHandle(runtime)
    cache_handle.all_context_pages_resident = False
    runtime.begin_forward()
    cache_handle.swap_in(
        req_id_per_token=torch.tensor([0], dtype=torch.int32, device=device),
        block_table=torch.tensor([[1]], dtype=torch.int32, device=device),
        logical_topk_indices=torch.tensor([[0]], dtype=torch.int32, device=device),
        block_size=block_size,
        num_valid_rows=_valid_rows(1),
    )
    torch.accelerator.synchronize()

    assert (runtime.index_group.device_global_indices[0] == -1).all()
    assert (runtime.index_group.device_global_indices[1] == block_size).any()


@requires_hisparse_ops
def test_hisparse_maps_speculative_rows_through_request_state():
    """Multiple verification rows for one request share its persistent state."""
    device = torch.device(DEVICE_TYPE)
    block_size, row_width = 64, 8
    runtime = _make_hisparse_runtime(
        top_k=1,
        device_buffer_size=2,
        max_num_reqs=1,
        row_width=row_width,
        block_size=block_size,
        max_swap_rows=4,
    )
    runtime.bind_source_cache(
        torch.zeros(1, block_size, row_width, dtype=torch.float32).pin_memory()
    )

    cache_handle = HiSparseCacheHandle(runtime)
    resident_table = torch.tensor([[1]], dtype=torch.int32, device=device)
    cache_handle.bind_cache(
        runtime.hot.cache.view(torch.int8),
        byte_offset=0,
        block_stride=block_size * row_width * torch.float32.itemsize,
        num_blocks=2,
        block_size=block_size,
        block_table=resident_table,
        slot_mapping=torch.arange(4, dtype=torch.int64, device=device),
    )
    cache_handle.all_context_pages_resident = True
    runtime.begin_forward()
    physical = cache_handle.swap_in(
        req_id_per_token=torch.zeros(4, dtype=torch.int32, device=device),
        block_table=torch.zeros((1, 1), dtype=torch.int32, device=device),
        logical_topk_indices=torch.zeros((4, 1), dtype=torch.int32, device=device),
        block_size=block_size,
        num_valid_rows=_valid_rows(4),
    )
    torch.accelerator.synchronize()

    assert physical.tolist() == [[block_size]] * 4


@requires_hisparse_ops
@pytest.mark.parametrize(
    "num_rows,row_stride,top_k",
    [(8, 128, 128), (4, 32, 128), (8, 2048, 2048)],
    ids=["disjoint", "overlap", "long_mtp"],
)
def test_hisparse_speculative_rows_resolve_host_misses_consistently(
    num_rows, row_stride, top_k
):
    """Verification rows of one request must agree on its hot-buffer slots.

    Each row resolved residency in its own thread block against the request's
    shared slot ownership and LRU, so rows missing on different host rows could
    claim the same free slot. One row's hot index then pointed at a slot holding
    another row's KV. Overlapping rows must share one slot per host row, and
    long_mtp's rows span several resolution passes.
    """
    device = torch.device(DEVICE_TYPE)
    block_size, row_width = 64, 64
    num_blocks = 2 * num_rows * top_k // block_size

    kv_pool = torch.randn(
        (num_blocks + 1, block_size, row_width), dtype=torch.float32
    ).pin_memory()
    flat_pool = kv_pool.reshape(-1, row_width)
    runtime = _make_hisparse_runtime(
        top_k=top_k,
        device_buffer_size=num_rows * top_k,
        max_num_reqs=1,
        row_width=row_width,
        block_size=block_size,
        max_swap_rows=num_rows,
    )
    runtime.bind_source_cache(kv_pool)
    cache = HiSparseCacheHandle(runtime)
    cache.all_context_pages_resident = False

    block_table = torch.arange(
        1, num_blocks + 1, dtype=torch.int32, device=device
    ).view(1, num_blocks)
    req_ids = torch.zeros(num_rows, dtype=torch.int32, device=device)
    rows = torch.arange(num_rows, dtype=torch.int32, device=device)[:, None]
    base = torch.arange(top_k, dtype=torch.int32, device=device)[None, :]
    for step in range(4):
        runtime.begin_forward()
        # Rows start row_stride apart (disjoint at row_stride == top_k) and
        # shift by half a top-k per step: all misses first, then a mix.
        topk = (rows * row_stride + base + step * (top_k // 2)) % (
            num_blocks * block_size
        )
        hot_indices = cache.swap_in(
            req_id_per_token=req_ids,
            block_table=block_table,
            logical_topk_indices=topk.clone(),
            block_size=block_size,
            num_valid_rows=_valid_rows(num_rows),
        )
        torch.accelerator.synchronize()

        global_ref = _triton_convert_reference_impl(
            req_ids, block_table, topk, block_size, top_k
        )
        gathered = runtime.hot.attention_cache.reshape(-1, row_width)[
            hot_indices.to(torch.long)
        ].cpu()
        torch.testing.assert_close(gathered, flat_pool[global_ref.cpu().to(torch.long)])


@requires_hisparse_ops
def test_hisparse_speculative_rows_count_a_shared_load_once():
    """A host row two verification rows select loads once; the reuse is a hit."""
    device = torch.device(DEVICE_TYPE)
    block_size, row_width, top_k = 64, 64, 128
    num_blocks = top_k // block_size
    kv_pool = torch.randn(
        (num_blocks + 1, block_size, row_width), dtype=torch.float32
    ).pin_memory()
    runtime = _make_hisparse_runtime(
        top_k=top_k,
        device_buffer_size=2 * top_k,
        max_num_reqs=1,
        row_width=row_width,
        block_size=block_size,
        max_swap_rows=2,
    )
    runtime.bind_source_cache(kv_pool)
    cache = HiSparseCacheHandle(runtime)
    cache.all_context_pages_resident = False
    block_table = torch.arange(
        1, num_blocks + 1, dtype=torch.int32, device=device
    ).view(1, num_blocks)
    topk = torch.arange(top_k, dtype=torch.int32, device=device).repeat(2, 1)

    runtime.begin_forward()
    cache.swap_in(
        req_id_per_token=torch.zeros(2, dtype=torch.int32, device=device),
        block_table=block_table,
        logical_topk_indices=topk,
        block_size=block_size,
        num_valid_rows=_valid_rows(2),
    )
    torch.accelerator.synchronize()

    assert runtime.index_group.swap_stats.cpu().tolist() == [top_k, top_k]


@requires_hisparse_ops
def test_hisparse_resolver_reads_current_request_ids():
    """The resolver must see this call's request ids, not the previous call's.

    It runs on the copy stream after waiting only on ``logical_topk_ready``,
    which the indexer records before attention writes the ids. An MTP drafter
    alternates four rows per request with one, so stale ids resolve rows
    against other requests' KV. Stalling the compute stream exposes the race.
    """
    device = torch.device(DEVICE_TYPE)
    block_size, row_width, top_k, blocks_per_req, num_reqs = 64, 16, 64, 4, 2
    kv_pool = torch.randn(
        (num_reqs * blocks_per_req + 1, block_size, row_width), dtype=torch.float32
    ).pin_memory()
    runtime = _make_hisparse_runtime(
        top_k=top_k,
        device_buffer_size=4 * top_k,
        max_num_reqs=num_reqs,
        row_width=row_width,
        block_size=block_size,
        max_swap_rows=4 * num_reqs,
    )
    runtime.bind_source_cache(kv_pool)
    ready = torch.Event()
    runtime.index_group.logical_topk_ready = ready
    cache = HiSparseCacheHandle(runtime)
    cache.all_context_pages_resident = False
    cache.source_block_table = torch.arange(
        1, num_reqs * blocks_per_req + 1, dtype=torch.int32, device=device
    ).view(num_reqs, blocks_per_req)

    index_group = object.__new__(HiSparseMLAIndexGroup)
    index_group.caches = [cache]
    index_group.physical_topk_indices = torch.empty(
        (4 * num_reqs + 1, top_k), dtype=torch.int32, device=device
    )

    # Four rows per request, then one per request plus a CUDA-graph padding
    # row, which maps to request 0 and must be skipped.
    for rows_per_req, num_padding in ((4, 0), (1, 1)):
        num_rows = num_reqs * rows_per_req
        req_ids = torch.arange(num_reqs, dtype=torch.int32, device=device)
        req_ids = torch.cat(
            [
                req_ids.repeat_interleave(rows_per_req),
                torch.zeros(num_padding, dtype=torch.int32, device=device),
            ]
        )
        topk = torch.stack(
            [
                torch.randperm(blocks_per_req * block_size, device=device)[:top_k]
                for _ in range(req_ids.numel())
            ]
        ).to(torch.int32)
        query_start_loc = torch.arange(
            0, num_rows + 1, rows_per_req, dtype=torch.int32, device=device
        )
        attn_metadata = SimpleNamespace(
            req_id_per_token=req_ids,
            query_start_loc=torch.cat([query_start_loc, query_start_loc[-1:]]),
            block_size=block_size,
        )
        runtime.begin_forward()
        ready.record(current_stream())
        torch.cuda._sleep(200_000_000)
        hot_indices = index_group.convert_logical_to_physical_topk(
            0, topk, attn_metadata, block_stride_rows=None, return_valid_counts=False
        )
        torch.accelerator.synchronize()

        assert (hot_indices[num_rows:] == -1).all()
        global_ref = _triton_convert_reference_impl(
            req_ids[:num_rows],
            cache.source_block_table,
            topk[:num_rows],
            block_size,
            top_k,
        )
        gathered = runtime.hot.attention_cache.reshape(-1, row_width)[
            hot_indices[:num_rows].to(torch.long)
        ].cpu()
        torch.testing.assert_close(
            gathered, kv_pool.reshape(-1, row_width)[global_ref.cpu().to(torch.long)]
        )


@requires_hisparse_ops
def test_hisparse_union_table_scales_with_union_bound():
    """Shared memory follows the per-request union bound, not the hot buffer.

    Sizing the table from device_buffer_size needs ~340 KB here and fails.
    """
    device = torch.device(DEVICE_TYPE)
    block_size, row_width, top_k = 64, 64, 128
    num_blocks = top_k // block_size
    kv_pool = torch.randn(
        (num_blocks + 1, block_size, row_width), dtype=torch.float32
    ).pin_memory()
    runtime = _make_hisparse_runtime(
        top_k=top_k,
        device_buffer_size=32768,
        max_union_rows=top_k,
        max_num_reqs=1,
        row_width=row_width,
        block_size=block_size,
        max_swap_rows=1,
    )
    runtime.bind_source_cache(kv_pool)
    cache = HiSparseCacheHandle(runtime)
    cache.all_context_pages_resident = False
    req_ids = torch.zeros(1, dtype=torch.int32, device=device)
    block_table = torch.arange(
        1, num_blocks + 1, dtype=torch.int32, device=device
    ).view(1, num_blocks)
    topk = torch.arange(top_k, dtype=torch.int32, device=device).view(1, top_k)

    runtime.begin_forward()
    hot_indices = cache.swap_in(
        req_id_per_token=req_ids,
        block_table=block_table,
        logical_topk_indices=topk,
        block_size=block_size,
        num_valid_rows=_valid_rows(req_ids.numel()),
    )
    torch.accelerator.synchronize()

    global_ref = _triton_convert_reference_impl(
        req_ids, block_table, topk, block_size, top_k
    )
    gathered = runtime.hot.attention_cache.reshape(-1, row_width)[
        hot_indices.to(torch.long)
    ].cpu()
    torch.testing.assert_close(
        gathered, kv_pool.reshape(-1, row_width)[global_ref.cpu().to(torch.long)]
    )


@requires_hisparse_ops
def test_hisparse_rejects_union_bound_beyond_shared_memory():
    """An oversized union bound fails at construction, not on the first forward."""
    with pytest.raises(ValueError, match="shared memory"):
        _make_hisparse_runtime(top_k=2048, device_buffer_size=32768)


@requires_hisparse_ops
def test_hisparse_resident_rows_bypass_hot_lru():
    device = torch.device(DEVICE_TYPE)
    block_size, row_width = 64, 8
    cache_handle = _make_hisparse_cache_handle(
        top_k=4,
        device_buffer_size=5,
        max_num_reqs=1,
        row_width=row_width,
        block_size=block_size,
    )
    raw = cache_handle.runtime.hot.cache.view(torch.int8)
    resident_table = torch.tensor([[1]], dtype=torch.int32, device=device)
    resident_slots = torch.tensor([block_size], dtype=torch.int64, device=device)
    cache_handle.bind_cache(
        raw,
        byte_offset=0,
        block_stride=block_size * row_width * torch.float32.itemsize,
        num_blocks=2,
        block_size=block_size,
        block_table=resident_table,
        slot_mapping=resident_slots,
    )

    host = torch.randn(1, block_size, row_width).pin_memory()
    cache_handle.runtime.bind_source_cache(host)
    resident_rows = host[0].to(device).add(7)
    cache_handle.runtime.hot.cache[1].copy_(resident_rows)
    source_table = torch.tensor([[5]], dtype=torch.int32, device=device)
    request_ids = torch.zeros(1, dtype=torch.int32, device=device)
    topk = torch.tensor([[1, 2, 3, 4]], dtype=torch.int32, device=device)
    lru_before = cache_handle.runtime.index_group.lru_slots.clone()

    cache_handle.runtime.begin_forward()
    indices, valid_counts = cache_handle.swap_in(
        req_id_per_token=request_ids,
        block_table=source_table,
        logical_topk_indices=topk,
        block_size=block_size,
        num_valid_rows=_valid_rows(request_ids.numel()),
        return_valid_counts=True,
    )
    torch.accelerator.synchronize()

    expected_indices = topk + block_size
    torch.testing.assert_close(indices, expected_indices)
    torch.testing.assert_close(
        valid_counts, torch.tensor([topk.shape[1]], dtype=torch.int32, device=device)
    )
    torch.testing.assert_close(cache_handle.runtime.index_group.lru_slots, lru_before)
    torch.testing.assert_close(
        cache_handle.runtime.hot.attention_cache.view(-1, row_width)[
            indices.to(torch.long)
        ],
        resident_rows[topk.to(torch.long)],
    )


@requires_hisparse_ops
def test_hisparse_swap_in_preserves_rows_across_eviction():
    device = torch.device(DEVICE_TYPE)
    block_size = 64
    row_width = 64
    num_blocks = 64
    top_k = 128
    buf = 256
    num_reqs = 4

    kv_pool = torch.randn(
        (num_blocks + 1, block_size, row_width), dtype=torch.float32
    ).pin_memory()
    flat_pool = kv_pool.reshape(-1, row_width)

    runtime = _make_hisparse_runtime(
        top_k=top_k,
        device_buffer_size=buf,
        max_num_reqs=num_reqs,
        row_width=row_width,
    )
    runtime.bind_source_cache(kv_pool)
    cache = HiSparseCacheHandle(runtime)
    cache.all_context_pages_resident = False
    cache.runtime.begin_forward()

    blocks_per_req = num_blocks // num_reqs
    block_table = torch.arange(
        1, num_blocks + 1, dtype=torch.int32, device=device
    ).view(num_reqs, blocks_per_req)
    req_ids = torch.arange(num_reqs, dtype=torch.int32, device=device)
    seq_len = blocks_per_req * block_size
    base = torch.arange(top_k, dtype=torch.int32, device=device)
    for step in range(3):
        cache.runtime.begin_forward()
        topk = torch.stack(
            [(base + step * top_k + row * 17) % seq_len for row in range(num_reqs)]
        )
        topk[:, -1] = -1

        hot_indices, valid_counts = cache.swap_in(
            req_id_per_token=req_ids,
            block_table=block_table,
            logical_topk_indices=topk.clone(),
            block_size=block_size,
            num_valid_rows=_valid_rows(req_ids.numel()),
            return_valid_counts=True,
        )
        torch.accelerator.synchronize()

        global_ref = _triton_convert_reference_impl(
            req_ids, block_table, topk, block_size, top_k
        )
        valid = hot_indices >= 0
        torch.testing.assert_close(
            valid_counts,
            (global_ref >= 0).sum(dim=1, dtype=torch.int32),
            rtol=0,
            atol=0,
        )
        gathered = cache.runtime.hot.attention_cache.reshape(-1, row_width)[
            hot_indices[valid].to(torch.long)
        ].cpu()
        expected = flat_pool[global_ref[valid].cpu().to(torch.long)]
        torch.testing.assert_close(gathered, expected)


@requires_hisparse_ops
def test_hisparse_multi_step_swaps_match_independent():
    """The shared LRU must retain every speculative step until followers run."""
    device = torch.device(DEVICE_TYPE)
    torch.manual_seed(0)
    block_size = 64
    row_width = 64
    num_blocks = 64
    top_k = 128
    buf = 3 * top_k
    num_reqs = 4

    kv_pool = torch.randn(
        (num_blocks, block_size, row_width), dtype=torch.float32
    ).pin_memory()

    def make(
        index_group: hisparse_runtime.HiSparseIndexGroup | None = None,
    ) -> HiSparseCacheHandle:
        runtime = _make_hisparse_runtime(
            top_k=top_k,
            max_num_reqs=num_reqs,
            device_buffer_size=buf,
            row_width=row_width,
            block_size=block_size,
            max_swap_rows=2 * num_reqs,
            index_group=index_group,
        )
        runtime.bind_source_cache(kv_pool)
        cache = HiSparseCacheHandle(runtime)
        cache.all_context_pages_resident = False
        return cache

    blocks_per_req = num_blocks // num_reqs
    block_table = torch.arange(num_blocks, dtype=torch.int32, device=device).view(
        num_reqs, blocks_per_req
    )
    req_ids = torch.arange(num_reqs, dtype=torch.int32, device=device)
    seq_len = blocks_per_req * block_size
    producer = make()
    shared = make(producer.runtime.index_group)
    indep = make()
    base = torch.arange(top_k, dtype=torch.int32, device=device)
    for iteration in range(2):
        producer.runtime.begin_forward()
        shared.runtime.begin_forward()
        indep.runtime.begin_forward()
        topks = [
            torch.stack(
                [
                    (base + (iteration * 2 + step) * top_k + row * 17) % seq_len
                    for row in range(num_reqs)
                ]
            )
            for step in range(2)
        ]
        kw = dict(
            req_id_per_token=req_ids,
            block_table=block_table,
            block_size=block_size,
            num_valid_rows=_valid_rows(req_ids.numel()),
        )
        producer_indices = [
            producer.swap_in(logical_topk_indices=topk.clone(), **kw) for topk in topks
        ]
        independent_indices = [
            indep.swap_in(logical_topk_indices=topk.clone(), **kw) for topk in topks
        ]
        follower_indices = [
            shared.swap_in(logical_topk_indices=topk.clone(), **kw) for topk in topks
        ]
        torch.accelerator.synchronize()

        for idx_full, idx_indep, idx_shared in zip(
            producer_indices, independent_indices, follower_indices
        ):
            torch.testing.assert_close(idx_shared, idx_full, rtol=0, atol=0)
            torch.testing.assert_close(idx_indep, idx_full, rtol=0, atol=0)
        torch.testing.assert_close(shared.runtime.hot.cache, indep.runtime.hot.cache)

        shared_rows = shared.runtime.hot.attention_cache.reshape(-1, row_width)
        host_rows = kv_pool.reshape(-1, row_width)
        for physical_rows, logical_rows in zip(follower_indices, topks):
            global_rows = _triton_convert_reference_impl(
                req_ids, block_table, logical_rows, block_size, top_k
            )
            valid = physical_rows >= 0
            torch.testing.assert_close(
                shared_rows[physical_rows[valid].to(torch.long)].cpu(),
                host_rows[global_rows[valid].cpu().to(torch.long)],
            )


@requires_hisparse_ops
def test_hisparse_multi_step_writes_request_major_output():
    device = torch.device(DEVICE_TYPE)
    block_size = 64
    row_width = 8
    num_reqs = 2
    query_len = 3
    top_k = 4
    cache = _make_hisparse_cache_handle(
        top_k=top_k,
        device_buffer_size=query_len * top_k,
        max_num_reqs=num_reqs,
        row_width=row_width,
        block_size=block_size,
        max_swap_rows=num_reqs * query_len,
    )
    host = torch.arange(
        (num_reqs + 1) * block_size * row_width, dtype=torch.float32
    ).view(num_reqs + 1, block_size, row_width)
    cache.runtime.bind_source_cache(host.pin_memory())
    block_table = torch.arange(1, num_reqs + 1, dtype=torch.int32, device=device).view(
        -1, 1
    )
    request_ids = torch.arange(num_reqs, dtype=torch.int32, device=device)
    logical = torch.tensor(
        [
            [[0, 1, 2, 3], [4, 5, 6, 7], [8, 9, 10, 11]],
            [[12, 13, 14, 15], [16, 17, 18, 19], [20, 21, 22, 23]],
        ],
        dtype=torch.int32,
        device=device,
    )
    physical = torch.full_like(logical, -1)
    valid_counts = torch.zeros((num_reqs, query_len), dtype=torch.int32, device=device)

    cache.runtime.begin_forward()
    for step in range(query_len):
        cache.swap_in(
            request_ids,
            block_table,
            logical[:, step],
            block_size=block_size,
            num_valid_rows=_valid_rows(request_ids.numel()),
            return_valid_counts=True,
            attention_indices_out=physical[:, step],
            valid_counts_out=valid_counts[:, step],
        )
    torch.accelerator.synchronize()

    torch.testing.assert_close(
        valid_counts,
        torch.full_like(valid_counts, top_k),
    )
    global_rows = _triton_convert_reference_impl(
        torch.arange(num_reqs, device=device, dtype=torch.int32).repeat_interleave(
            query_len
        ),
        block_table,
        logical.view(num_reqs * query_len, top_k),
        block_size,
        top_k,
    )
    hot_rows = cache.runtime.hot.attention_cache.reshape(-1, row_width)
    host_rows = host.reshape(-1, row_width)
    torch.testing.assert_close(
        hot_rows[physical.view(num_reqs * query_len, top_k).to(torch.long)].cpu(),
        host_rows[global_rows.cpu().to(torch.long)],
    )


@requires_hisparse_ops
def test_hisparse_kv_update_writes_resident_cache():
    device = torch.device(DEVICE_TYPE)
    block_size = 4
    row_width = 8
    cache_handle = _make_hisparse_cache_handle(
        max_num_reqs=1,
        row_width=row_width,
        block_size=block_size,
    )
    resident_slots = torch.tensor([4, 5, -1], dtype=torch.int64, device=device)
    cache_handle.bind_cache(
        cache_handle.runtime.hot.cache.view(torch.int8),
        byte_offset=0,
        block_stride=block_size * row_width * torch.float32.itemsize,
        num_blocks=cache_handle.runtime.hot.cache.shape[0],
        block_size=block_size,
        block_table=torch.tensor([[1]], dtype=torch.int32, device=device),
        slot_mapping=resident_slots,
    )
    slots = torch.tensor([3, 7, -1], dtype=torch.int64, device=device)
    kv_c = torch.randn(8, row_width - 2, device=device)
    k_pe = torch.randn(8, 1, 2, device=device)
    cache_handle.num_actual_tokens = slots.numel()
    cache_handle.decode_batch = False
    source_cache = torch.zeros_like(cache_handle.view.cache)
    impl = object.__new__(FlashMLASparseImpl)
    layer = SimpleNamespace(
        hisparse_cache=cache_handle,
        use_pcp=False,
        pcp_shard_decode_requests=False,
        impl=impl,
    )
    mla_attention.MLAAttention.update_kv_cache(
        layer,
        kv_c,
        k_pe,
        source_cache,
        slots,
        SimpleNamespace(num_decode_tokens=0),
        "auto",
        torch.tensor(1.0, device=device),
    )
    torch.accelerator.synchronize()

    expected = torch.cat([kv_c[:2], k_pe[:2, 0]], dim=-1).cpu()
    torch.testing.assert_close(
        cache_handle.view.cache.view(-1, row_width)[resident_slots[:2]],
        expected.to(device),
    )
    torch.testing.assert_close(source_cache, torch.zeros_like(source_cache))


@requires_hisparse_ops
def test_hisparse_row_dma_copies_discontiguous_spans_across_layers():
    device = torch.device(DEVICE_TYPE)
    block_size = 4
    row_width = 8
    num_rows = 16
    resident_caches = tuple(
        (
            torch.arange(num_rows * row_width, dtype=torch.uint8, device=device)
            .add_(layer_index * 17)
            .view(num_rows // block_size, block_size, row_width)
        )
        for layer_index in range(2)
    )
    host_caches = tuple(
        torch.zeros((num_rows, row_width), dtype=torch.uint8, pin_memory=True)
        for _ in resident_caches
    )
    worker = object.__new__(HiSparseConnectorWorker)
    worker.is_host_writer = True
    worker.kernel_block_size = block_size
    worker.resident_caches = resident_caches
    worker.host_caches = host_caches
    worker.cache_handles = [
        SimpleNamespace(
            decode_batch=True,
            runtime=SimpleNamespace(resident_source_index=0),
        )
        for _ in resident_caches
    ]
    worker.dma_stream = torch.cuda.Stream(device=device)
    worker.host_write_event = torch.Event()
    worker._dma_free_descriptors = []
    worker._pending_dma_descriptors = deque()
    worker._pending_transfer_events = deque()
    worker._enqueued_transfer_ids = []
    worker._dma_submitted = False
    worker._set_row_mirrors(
        (
            SparseKVRowMirror((1,), 5, 2),
            SparseKVRowMirror((8,), 10, 4),
        )
    )

    worker._enqueue_row_dma(range(2))
    current_stream().wait_event(worker.host_write_event)
    torch.accelerator.synchronize()

    for resident, host in zip(resident_caches, host_caches):
        flat_resident = resident.view(num_rows, row_width).cpu()
        torch.testing.assert_close(host[5:7], flat_resident[1:3])
        torch.testing.assert_close(host[10:14], flat_resident[8:12])


@requires_hisparse_ops
def test_hisparse_remaps_strided_hma_rows_for_attention():
    device = torch.device(DEVICE_TYPE)
    block_size, row_width = 4, 8
    runtime = _make_hisparse_runtime(
        max_num_reqs=1,
        block_size=block_size,
        row_width=row_width,
    )
    page_elements = block_size * row_width
    stride_elements = 3 * page_elements
    byte_offset = page_elements * torch.float32.itemsize
    raw = torch.zeros(2 * stride_elements, dtype=torch.float32, device=device).view(
        torch.int8
    )
    runtime.bind_hot_cache(
        raw,
        byte_offset=byte_offset,
        block_stride=stride_elements * torch.float32.itemsize,
        num_blocks=2,
        block_size=block_size,
        block_table=torch.tensor([[0, 1]], dtype=torch.int32, device=device),
    )

    source = (
        torch.arange(36 * row_width, dtype=torch.float32)
        .view(9, block_size, row_width)
        .pin_memory()
    )
    runtime.bind_source_cache(source)
    cache = HiSparseCacheHandle(runtime)
    cache.all_context_pages_resident = False
    runtime.begin_forward()
    attention_indices = cache.swap_in(
        req_id_per_token=torch.tensor([0], dtype=torch.int32, device=device),
        block_table=torch.arange(1, 9, dtype=torch.int32, device=device).view(1, 8),
        logical_topk_indices=torch.tensor(
            [[0, 1, 2, 3]], dtype=torch.int32, device=device
        ),
        block_size=block_size,
        num_valid_rows=_valid_rows(1),
    )

    assert runtime.hot.attention_cache is not None
    assert runtime.hot.attention_cache.is_contiguous()
    physical_indices = runtime.index_group.shared_topk.device_topk_rows[0]
    expected = (
        physical_indices // block_size * (stride_elements // row_width)
        + physical_indices % block_size
    )
    torch.testing.assert_close(attention_indices[0], expected)


@requires_hisparse_ops
def test_hisparse_gather_zeroes_unaligned_destination():
    """Invalid host rows are safely zeroed even at a byte-unaligned address."""
    device = torch.device(DEVICE_TYPE)
    row_bytes = 16
    host_cache = torch.ones(1, row_bytes, dtype=torch.uint8).pin_memory()
    storage = torch.full((2 * row_bytes + 1,), 7, dtype=torch.uint8, device=device)
    hot_cache = torch.as_strided(
        storage,
        size=(2, 1, row_bytes),
        stride=(row_bytes + 1, row_bytes, 1),
    )
    global_indices = torch.tensor([[1]], dtype=torch.int32, device=device)
    hot_indices = torch.tensor([[1]], dtype=torch.int32, device=device)
    miss_mask = torch.ones_like(global_indices)

    torch.ops._C_cache_ops.hisparse_gather_plan(
        host_cache,
        hot_cache,
        global_indices,
        hot_indices,
        miss_mask,
        None,
    )
    torch.accelerator.synchronize()

    torch.testing.assert_close(hot_cache[1, 0], torch.zeros_like(hot_cache[1, 0]))
    assert storage[row_bytes].item() == 7


@requires_hisparse_ops
def test_hisparse_gather_rejects_short_attention_block_stride():
    """Reject a stride that would let the gather kernel cross block bounds."""
    device = torch.device(DEVICE_TYPE)
    host_cache = torch.ones(1, 16, dtype=torch.uint8).pin_memory()
    hot_cache = torch.empty((1, 2, 16), dtype=torch.uint8, device=device)
    global_indices = torch.zeros((1, 1), dtype=torch.int32, device=device)
    hot_indices = torch.zeros_like(global_indices)
    miss_mask = torch.ones_like(global_indices)
    attention_indices = torch.empty_like(global_indices)

    with pytest.raises(RuntimeError, match="stride must cover one hot block"):
        torch.ops._C_cache_ops.hisparse_gather_plan(
            host_cache,
            hot_cache,
            global_indices,
            hot_indices,
            miss_mask,
            None,
            attention_indices,
            1,
        )


@requires_hisparse_ops
def test_hisparse_newest_write_and_recycled_slot_invalidation():
    """Newest writes clamp padding and invalidated rows are loaded again."""
    device = torch.device(DEVICE_TYPE)
    block_size = 4
    row_width = 8
    num_blocks = 8

    kv_pool = (
        torch.arange(num_blocks * block_size * row_width, dtype=torch.float32)
        .view(num_blocks, block_size, row_width)
        .pin_memory()
    )
    flat_pool = kv_pool.reshape(-1, row_width)

    cache_handle = _make_hisparse_cache_handle(block_size=block_size)
    cache_handle.runtime.eager_host_mirror = True

    block_table = torch.tensor([[2, 0, 4]], dtype=torch.int32, device=device)
    resident_slots = torch.tensor([4, -1], dtype=torch.int64, device=device)
    cache_handle.bind_cache(
        cache_handle.runtime.hot.cache.view(torch.int8),
        byte_offset=0,
        block_stride=block_size * row_width * torch.float32.itemsize,
        num_blocks=cache_handle.runtime.hot.cache.shape[0],
        block_size=block_size,
        block_table=torch.tensor([[0, 0, 1]], dtype=torch.int32, device=device),
        slot_mapping=resident_slots,
    )
    req_ids = torch.tensor([0], dtype=torch.int32, device=device)
    newest_global = 4 * block_size
    padded_global = 1
    padded_row = flat_pool[padded_global].clone()
    slot_mapping = torch.tensor(
        [newest_global, padded_global], dtype=torch.int64, device=device
    )
    request_state_indices = torch.tensor([0, -1], dtype=torch.int32, device=device)
    cache_handle.runtime.request_state_indices = request_state_indices

    kv_c = torch.randn(3, row_width - 2, device=device)
    k_pe = torch.randn(3, 1, 2, device=device)
    cache_handle.runtime.bind_source_cache(kv_pool)
    cache_handle.num_actual_tokens = slot_mapping.numel()
    ops.concat_and_cache_mla(
        kv_c[:1],
        k_pe[:1, 0],
        cache_handle.view.cache,
        resident_slots[:1],
        kv_cache_dtype="auto",
        scale=torch.tensor(1.0, device=device),
    )
    torch.accelerator.synchronize()
    expected_row = torch.cat([kv_c[0], k_pe[0, 0]]).cpu()
    flat_pool[newest_global].copy_(expected_row)
    torch.testing.assert_close(flat_pool[newest_global], expected_row)
    torch.testing.assert_close(flat_pool[padded_global], padded_row)
    flat_hot = cache_handle.runtime.hot.cache.view(-1, row_width)
    torch.testing.assert_close(flat_hot[resident_slots[0]].cpu(), expected_row)

    topk = torch.tensor([[0, -1, -1, -1]], dtype=torch.int32, device=device)
    cache_handle.all_context_pages_resident = False
    cache_handle.runtime.begin_forward()
    hot_indices = cache_handle.swap_in(
        req_id_per_token=req_ids,
        block_table=block_table,
        logical_topk_indices=topk,
        block_size=block_size,
        num_valid_rows=_valid_rows(req_ids.numel()),
    )
    torch.accelerator.synchronize()
    stale_hot_slot = hot_indices[0, 0].item()
    stale_row = flat_hot[stale_hot_slot].clone()

    flat_pool[8] += 1000
    cache_handle.runtime.invalidate_written_slots(
        torch.tensor([8], dtype=torch.int64, device=device), req_ids
    )
    cache_handle.runtime.begin_forward()
    hot_indices = cache_handle.swap_in(
        req_id_per_token=req_ids,
        block_table=block_table,
        logical_topk_indices=topk,
        block_size=block_size,
        num_valid_rows=_valid_rows(req_ids.numel()),
    )
    torch.accelerator.synchronize()
    idx = hot_indices.cpu().tolist()[0][0]
    assert not torch.equal(flat_hot[idx], stale_row)
    torch.testing.assert_close(flat_hot[idx].cpu(), flat_pool[8])


@requires_hisparse_ops
def test_hisparse_mixed_batch_bf16_row_split(
    default_vllm_config, dist_init, workspace_init
):
    """Host-resident mixed batch on the bf16 path is row-split.

    Two long-context speculative-decode requests + one short local-prefill
    chunk: every decode step must be served from the bounded hot buffer before
    it is reused, while only the prefill rows' blocks are staged host->GPU.
    """
    ok, reason = flashmla.is_flashmla_sparse_supported()
    if not ok:
        pytest.skip(reason)

    device = torch.device(DEVICE_TYPE)
    dtype = torch.bfloat16
    torch.manual_seed(0)

    num_heads = 64
    kv_lora_rank = 512
    qk_nope_head_dim = 128
    qk_rope_head_dim = 64
    v_head_dim = 128
    head_size = kv_lora_rank + qk_rope_head_dim
    topk_tokens = 128
    block_size = 64

    # Long decode contexts + a short prefill chunk (router shortcut shape).
    batch_spec = BatchSpec(seq_lens=[2048, 2048, 192], query_lens=[2, 2, 64])
    max_seqlen = max(batch_spec.seq_lens)
    total_cache_tokens = sum(batch_spec.seq_lens)
    total_tokens = batch_spec.compute_num_tokens()

    vllm_config = create_vllm_config(
        model_name="deepseek-ai/DeepSeek-V2-Lite-Chat",
        tensor_parallel_size=1,
        max_model_len=max_seqlen,
        num_gpu_blocks=max(2048, cdiv(total_cache_tokens, block_size) + 1),
        block_size=block_size,
        hf_config_override={
            "index_topk": topk_tokens,
            "attn_module_list_cfg": [{"topk_tokens": topk_tokens}],
        },
    )
    vllm_config.attention_config.hisparse_config = HiSparseConfig(
        device_buffer_size=2 * topk_tokens,
    )
    vllm_config.speculative_config = SpeculativeConfig(
        method="ngram", num_speculative_tokens=1
    )
    model_config = vllm_config.model_config
    model_config.hf_text_config = SimpleNamespace(
        index_topk=topk_tokens,
        q_lora_rank=None,
        kv_lora_rank=kv_lora_rank,
        qk_nope_head_dim=qk_nope_head_dim,
        qk_rope_head_dim=qk_rope_head_dim,
        v_head_dim=v_head_dim,
        model_type="deepseek_v2",
    )
    model_config.dtype = dtype
    model_config.get_num_attention_heads = MethodType(
        lambda self, parallel_config: num_heads, model_config
    )
    model_config.get_num_kv_heads = MethodType(
        lambda self, parallel_config: 1, model_config
    )
    model_config.get_head_size = MethodType(lambda self: head_size, model_config)
    model_config.get_sliding_window = MethodType(lambda self: None, model_config)
    # create_hisparse_cache_handle sizes hot buffers per layer.
    model_config.get_num_layers = MethodType(
        lambda self, parallel_config: 1, model_config
    )

    kv_cache_spec = create_standard_kv_cache_spec(vllm_config)
    common_attn_metadata = create_common_attn_metadata(
        batch_spec,
        vllm_config.cache_config.block_size,
        device,
        arange_block_indices=True,
    )
    # Prepopulate every position of every sequence so the forward needs no
    # KV-cache update (the row split itself is what is under test).
    kv_c_contexts = [
        torch.rand(s_len, kv_lora_rank, dtype=dtype, device=device)
        for s_len in batch_spec.seq_lens
    ]
    k_pe_contexts = [
        torch.rand(s_len, 1, qk_rope_head_dim, dtype=dtype, device=device)
        for s_len in batch_spec.seq_lens
    ]
    kv_cache = create_and_prepopulate_kv_cache(
        kv_c_contexts=kv_c_contexts,
        k_pe_contexts=k_pe_contexts,
        block_size=vllm_config.cache_config.block_size,
        head_size=head_size,
        dtype=dtype,
        device=device,
        num_blocks=vllm_config.cache_config.num_gpu_blocks,
        common_attn_metadata=common_attn_metadata,
        randomize_blocks=False,
        kv_cache_dtype="auto",
    )

    prefill_backend = get_mla_prefill_backend(vllm_config)(
        num_heads=num_heads,
        scale=1.0 / math.sqrt(head_size),
        kv_lora_rank=kv_lora_rank,
        qk_nope_head_dim=qk_nope_head_dim,
        qk_rope_head_dim=qk_rope_head_dim,
        v_head_dim=v_head_dim,
        vllm_config=vllm_config,
    )
    vllm_config.compilation_config.static_forward_context["placeholder"] = (
        SimpleNamespace(prefill_backend=prefill_backend, impl=SimpleNamespace())
    )

    builder_cls = FlashMLASparseBackend.get_builder_cls()
    builder = builder_cls(kv_cache_spec, ["placeholder"], vllm_config, device)
    metadata = builder.build(
        common_prefix_len=0, common_attn_metadata=common_attn_metadata
    )
    assert isinstance(metadata.prefill, SparseMLAPrefillMetadata)

    # Per-token sparse indices bounded by each token's position, with unique
    # offsets and -1 padding (same construction as the decode parity test).
    positions: list[int] = []
    for i in range(batch_spec.batch_size):
        ctx_len = batch_spec.seq_lens[i] - batch_spec.query_lens[i]
        positions.extend(ctx_len + q_idx for q_idx in range(batch_spec.query_lens[i]))
    sparse_indices = torch.empty(
        total_tokens, topk_tokens, dtype=torch.int32, device=device
    )
    for tok_idx in range(total_tokens):
        max_valid_idx = positions[tok_idx]
        offset = tok_idx * 7  # Prime number for varied offsets
        num_valid = min(topk_tokens // 2, max_valid_idx + 1)
        valid_range = torch.arange(num_valid, device=device, dtype=torch.int32)
        sparse_indices[tok_idx] = torch.cat(
            [
                (valid_range + offset) % (max_valid_idx + 1),
                torch.full(
                    (topk_tokens - num_valid,), -1, device=device, dtype=torch.int32
                ),
            ]
        )

    q = torch.rand(total_tokens, num_heads, head_size, dtype=dtype, device=device)
    mock_indexer = SimpleNamespace(
        topk_indices_buffer=sparse_indices, topk_tokens=topk_tokens
    )

    impl_cls = FlashMLASparseBackend.get_impl_cls()
    with set_current_vllm_config(vllm_config):
        impl = impl_cls(
            num_heads=num_heads,
            head_size=head_size,
            scale=1.0 / math.sqrt(head_size),
            num_kv_heads=1,
            alibi_slopes=None,
            sliding_window=None,
            kv_cache_dtype="auto",
            logits_soft_cap=None,
            attn_type="decoder",
            kv_sharing_target_layer_name=None,
            q_lora_rank=None,
            kv_lora_rank=kv_lora_rank,
            qk_nope_head_dim=qk_nope_head_dim,
            qk_rope_head_dim=qk_rope_head_dim,
            qk_head_dim=qk_nope_head_dim + qk_rope_head_dim,
            v_head_dim=v_head_dim,
            kv_b_proj=None,
            indexer=mock_indexer,
        )
    assert isinstance(impl.index_group, HiSparseMLAIndexGroup)
    cache_handle = impl.index_group.cache(impl.index_group_index)
    blocks_per_request = cdiv(cache_handle.runtime.region_stride, block_size)
    num_hot_blocks = cache_handle.runtime.max_num_reqs * blocks_per_request
    hot_cache = torch.zeros(
        num_hot_blocks * block_size * head_size,
        dtype=dtype,
        device=device,
    ).view(torch.int8)
    cache_handle.runtime.bind_hot_cache(
        hot_cache,
        byte_offset=0,
        block_stride=block_size * head_size * dtype.itemsize,
        num_blocks=num_hot_blocks,
        block_size=block_size,
        block_table=torch.arange(num_hot_blocks, dtype=torch.int32, device=device).view(
            cache_handle.runtime.max_num_reqs, blocks_per_request
        ),
    )
    cache_handle.runtime.request_state_indices = torch.arange(
        cache_handle.runtime.max_num_reqs, dtype=torch.int32, device=device
    )
    impl.prepare_for_batch(metadata)

    # Device-resident reference: the whole batch converted against the full
    # block table and run as one kernel call over the GPU cache.
    ref_topk, ref_topk_length = triton_convert_req_index_to_global_index(
        metadata.req_id_per_token,
        metadata.block_table,
        sparse_indices,
        BLOCK_SIZE=metadata.block_size,
        NUM_TOPK_TOKENS=topk_tokens,
        return_valid_counts=True,
    )
    reference, _ = impl._bf16_flash_mla_kernel(q, kv_cache, ref_topk, ref_topk_length)

    # Host-resident pool with identical contents.
    kv_pool = kv_cache.squeeze(1).cpu().pin_memory()
    cache_handle.runtime.bind_source_cache(kv_pool)
    cache_handle.source_block_table = metadata.block_table
    cache_handle.all_context_pages_resident = False

    staging_calls = []
    original_gather = cache_handle.runtime.gather_prefill_cache

    def spy_gather(self, kv, plan, resident_cache=None):
        staged = original_gather(kv, plan, resident_cache)
        staging_calls.append((plan, staged.shape))
        return staged

    cache_handle.runtime.gather_prefill_cache = MethodType(
        spy_gather, cache_handle.runtime
    )

    backend_output, _ = impl._forward_bf16_kv(
        q, kv_pool, sparse_indices, metadata, q.shape[1]
    )
    torch.accelerator.synchronize()

    # Only the prefill rows' blocks were staged: the decode rows' 2048-token
    # contexts (32 blocks each) must stay off the staging gather.
    assert len(staging_calls) == 1
    plan, staged_shape = staging_calls[0]
    assert plan is metadata.prefill.host_staging_plan
    prefill_blocks = cdiv(batch_spec.seq_lens[-1], block_size)
    assert staged_shape[0] <= prefill_blocks + 1  # +1: block-0 tail padding

    torch.testing.assert_close(backend_output, reference, rtol=0.01, atol=0.01)


def test_hisparse_prefill_staging_remap():
    """Compacted staging references the same host rows as direct indexing."""
    block_size = 4
    block_table = torch.tensor(
        [[5, 2, -1, -1], [2, 7, 3, -1], [9, -1, -1, -1]], dtype=torch.int32
    )

    new_bt, row_ids = hisparse_prefill_staging_remap(block_table, block_size)

    assert new_bt.shape == block_table.shape
    assert row_ids.dtype == torch.int32 and new_bt.dtype == torch.int32
    n_unique = len({0, 2, 3, 5, 7, 9})  # -1 clamps to block 0
    flat_rows = row_ids.flatten()
    assert (flat_rows >= 0).sum() == n_unique * block_size
    for i in range(block_table.shape[0]):
        for j in range(block_table.shape[1]):
            orig = max(int(block_table[i, j]), 0)
            staged = int(new_bt[i, j])
            for k in range(block_size):
                assert int(flat_rows[staged * block_size + k]) == orig * block_size + k


def test_hisparse_prefill_staging_plan_masks_unused_blocks():
    block_table = torch.tensor([[5, 2, 7], [9, 3, 4]], dtype=torch.int32)
    plan = build_hisparse_prefill_staging_plan(
        block_table,
        seq_lens=torch.tensor([5, 8], dtype=torch.int32),
        block_size=4,
        staging_block_capacity=4,
    )

    assert (plan.block_table[:, 2] == 0).all()
    valid_rows = plan.row_ids[plan.row_ids >= 0]
    assert set((valid_rows[::4] // 4).tolist()) == {0, 2, 3, 5, 9}
    for row, used in enumerate((2, 2)):
        for column in range(used):
            staged_block = int(plan.block_table[row, column])
            host_block = int(block_table[row, column])
            assert int(valid_rows[staged_block * 4]) == host_block * 4


def test_hisparse_prefill_staging_plan_resolves_resident_sources():
    """Per-page resident hits become device sources; misses stay host DMAs."""
    block_size = 4
    resident_block_size = 2
    block_table = torch.tensor([[5, 2, 0], [9, 3, 0]], dtype=torch.int32)
    seq_lens = torch.tensor([5, 8], dtype=torch.int32)
    plan = build_hisparse_prefill_staging_plan(
        block_table, seq_lens, block_size, staging_block_capacity=4
    )
    # Two resident pages per host block; 0 entries are null (not resident).
    resident_table = torch.tensor(
        [[11, 12, 0, 13, 0, 0], [21, 0, 22, 23, 0, 0]], dtype=torch.int32
    )

    state_indices = torch.arange(2, dtype=torch.int32)
    plan.ensure_gpu_sources(resident_table, state_indices, resident_block_size)

    assert plan.gpu_row_ids is not None
    unique_hosts = (plan.row_ids[0].view(-1, block_size)[:, 0] // block_size).tolist()
    gpu_rows = plan.gpu_row_ids[0].view(-1, block_size)
    reps = {5: (0, 0), 2: (0, 1), 9: (1, 0), 3: (1, 1)}
    for u, host_id in enumerate(unique_hosts):
        for t in range(block_size):
            if host_id < 0:
                assert int(gpu_rows[u, t]) == -1
                continue
            if host_id == 0:
                assert int(gpu_rows[u, t]) == -1
                continue
            row, col = reps[host_id]
            page = t // resident_block_size
            res_block = int(resident_table[row, col * 2 + page])
            expected = (
                res_block * resident_block_size + t % resident_block_size
                if res_block > 0
                else -1
            )
            assert int(gpu_rows[u, t]) == expected
    valid_rows = plan.row_ids >= 0
    torch.testing.assert_close(
        plan.miss_mask,
        ((plan.gpu_row_ids < 0) & valid_rows).int(),
        check_dtype=False,
    )

    plan.ensure_gpu_sources(
        torch.zeros_like(resident_table), state_indices, resident_block_size
    )

    assert plan.gpu_row_ids is not None
    assert (plan.gpu_row_ids == -1).all()


def test_hisparse_prefill_resident_sources_follow_each_resident_group():
    """Layers of different resident groups share one batch staging plan.

    Each group's layers must stage from that group's resident blocks. Keying
    the plan's cached resident sources on a per-call batch gather let a later
    group's gather reuse the freed address of an earlier one, so its layers
    read the earlier group's block ids.
    """
    block_size = 4
    plan = build_hisparse_prefill_staging_plan(
        torch.tensor([[5, 2]], dtype=torch.int32),
        torch.tensor([8], dtype=torch.int32),
        block_size,
        staging_block_capacity=2,
    )
    # The prefill runs in request state row 1; one table holds both groups.
    # Rows are wide so that a cache keyed on a freed per-call gather of a row
    # would see the next gather reuse its address.
    state_rows = torch.zeros((4, 2, 1024), dtype=torch.int32)
    state_rows[1, 0, :2] = torch.tensor([11, 12])
    state_rows[1, 1, :2] = torch.tensor([21, 22])
    state_indices = torch.tensor([1], dtype=torch.int32)
    host_ids = (plan.row_ids[0].view(-1, block_size)[:, 0] // block_size).tolist()

    for group in (0, 1, 0, 1):
        plan.ensure_gpu_sources(state_rows[:, group], state_indices, block_size)
        assert plan.gpu_row_ids is not None
        gpu_blocks = plan.gpu_row_ids[0].view(-1, block_size)[:, 0] // block_size
        staged = {
            host_id: gpu_block
            for host_id, gpu_block in zip(host_ids, gpu_blocks.tolist())
            if host_id > 0
        }
        first_page, second_page = state_rows[1, group, :2].tolist()
        assert staged == {5: first_page, 2: second_page}


@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_hisparse_gather_prefill_cache_prefers_resident_rows():
    """Staged rows come from the resident cache when a shadow page exists."""
    if not _has_hisparse_ops():
        pytest.skip("hisparse CUDA ops unavailable")
    device = torch.device("cuda")
    block_size, resident_block_size, row_width = 4, 2, 16
    block_table = torch.tensor([[5, 2, 0], [9, 3, 0]], dtype=torch.int32, device=device)
    seq_lens = torch.tensor([5, 8], dtype=torch.int32, device=device)
    plan = build_hisparse_prefill_staging_plan(
        block_table, seq_lens, block_size, staging_block_capacity=4
    )
    resident_table = torch.tensor(
        [[11, 12, 0, 13, 0, 0], [21, 0, 22, 23, 0, 0]],
        dtype=torch.int32,
        device=device,
    )
    plan.ensure_gpu_sources(
        resident_table,
        torch.arange(2, dtype=torch.int32, device=device),
        resident_block_size,
    )

    num_host_blocks, num_res_blocks = 10, 24
    host_cache = (
        torch.arange(num_host_blocks * block_size * row_width, dtype=torch.float32)
        .view(num_host_blocks, block_size, row_width)
        .pin_memory()
    )
    resident_cache = (
        (
            -torch.arange(
                num_res_blocks * resident_block_size * row_width, dtype=torch.float32
            )
            - 1.0
        )
        .view(num_res_blocks, resident_block_size, row_width)
        .to(device)
    )

    staged = HiSparseRuntime.gather_prefill_cache(
        None, host_cache, plan, resident_cache=resident_cache
    )

    staged_flat = staged.view(-1, row_width).cpu()
    host_flat = host_cache.view(-1, row_width)
    resident_flat = resident_cache.reshape(-1, row_width).cpu()
    gpu_rows = plan.gpu_row_ids[0].cpu()
    host_rows = plan.row_ids[0].cpu()
    for i in range(staged_flat.shape[0]):
        if int(host_rows[i]) < 0:
            assert int(plan.miss_mask[0, i]) == 0
            continue
        expected = (
            resident_flat[int(gpu_rows[i])]
            if int(gpu_rows[i]) >= 0
            else host_flat[int(host_rows[i])]
        )
        torch.testing.assert_close(staged_flat[i], expected)


def test_hisparse_fp8_decode_resolves_rows_once_then_runs_batched_attention():
    """Verification rows resolve in one swap_in; FP8 attention sees them batched."""
    device = torch.device("cpu")
    num_decodes = 2
    query_len = 3
    num_tokens = num_decodes * query_len
    q = torch.randn(num_tokens, 2, 4, device=device)
    topk = (
        torch.arange(num_tokens, dtype=torch.int32, device=device)
        .view(-1, 1)
        .expand(-1, 4)
    )
    kernel_shapes: list[torch.Size] = []

    def swap_in(req_ids, *, logical_topk_indices, **kwargs):  # noqa: ARG001
        return logical_topk_indices + 10

    def run_kernel(self, *, q, **kwargs):  # noqa: ARG001
        kernel_shapes.append(q.shape)
        return q[..., :1], None

    cache = SimpleNamespace(
        runtime=SimpleNamespace(
            hot=SimpleNamespace(attention_cache=torch.empty(1, device=device)),
            max_swap_rows=num_tokens,
        ),
        source_block_table=torch.empty(
            num_decodes, 1, dtype=torch.int32, device=device
        ),
        swap_in=MagicMock(side_effect=swap_in),
        all_context_pages_resident=True,
        num_decode_tokens=num_tokens,
        num_actual_tokens=num_tokens,
    )
    index_group = object.__new__(HiSparseMLAIndexGroup)
    index_group.caches = [cache]
    index_group.physical_topk_indices = torch.empty(
        (num_tokens + 1, 4), dtype=torch.int32, device=device
    )
    impl = SimpleNamespace(
        kv_lora_rank=1,
        index_group=index_group,
        index_group_index=0,
    )
    impl._fp8_flash_mla_kernel = MethodType(run_kernel, impl)
    metadata = SimpleNamespace(
        query_start_loc=torch.arange(
            0, num_tokens + 1, query_len, dtype=torch.int32, device=device
        ),
        req_id_per_token=torch.arange(
            num_decodes, dtype=torch.int32, device=device
        ).repeat_interleave(query_len),
        block_size=64,
    )

    output = FlashMLASparseImpl._host_backed_fp8_decode(
        impl, q, topk, metadata, SimpleNamespace(), num_decodes
    )

    cache.swap_in.assert_called_once()
    torch.testing.assert_close(
        cache.swap_in.call_args.args[0], metadata.req_id_per_token
    )
    torch.testing.assert_close(
        cache.swap_in.call_args.kwargs["logical_topk_indices"], topk
    )
    assert kernel_shapes == [(num_decodes, query_len, 2, 4)]
    assert output.shape == (num_tokens, 2, 1)


def test_hisparse_keeps_speculative_rows_as_decodes():
    """HiSparse limits decodes to verification requests, not short extends.

    FlashAttn sparse MLA used to classify MTP verification as prefill under
    HiSparse, whose residency-dependent route a FULL CUDA graph freezes at
    capture.
    """
    builder = object.__new__(FlashAttnMLASparseMetadataBuilder)
    builder.vllm_config = SimpleNamespace(
        attention_config=SimpleNamespace(hisparse_config=object()),
        speculative_config=SimpleNamespace(
            num_speculative_tokens=3, parallel_drafting=False
        ),
        parallel_config=SimpleNamespace(decode_context_parallel_size=1),
        use_v2_model_runner=True,
    )

    builder._init_reorder_batch_threshold(256, supports_spec_as_decode=True)

    assert builder.reorder_batch_threshold == 4


@pytest.mark.skipif(
    not current_platform.is_device_capability_family(90),
    reason="FlashAttention MLA requires Hopper",
)
@pytest.mark.parametrize("use_cuda_graph", [False, True])
@pytest.mark.parametrize("block_stride_rows", [64, 128])
@pytest.mark.parametrize(
    "query_len,num_decode_tokens,max_decode_rows,num_layers",
    [
        pytest.param(1, 2, 2, 4, id="decode"),
        pytest.param(4, 8, 8, 4, id="mtp4"),
        pytest.param(6, 12, 12, 4, id="mtp6"),
        pytest.param(4, 8, 2, 4, id="scratch_overflow"),
        pytest.param(4, 4, 8, 4, id="mixed"),
        pytest.param(4, 0, 8, 4, id="prefill"),
        pytest.param(4, 8, 8, 1, id="single_layer"),
    ],
)
def test_flashattn_shared_indices_match_independent_conversion(
    monkeypatch,
    query_len,
    num_decode_tokens,
    max_decode_rows,
    num_layers,
    block_stride_rows,
    use_cuda_graph,
):
    """Reuse must preserve attention outputs as requests and pages change."""
    device = torch.device(DEVICE_TYPE)
    num_tokens, block_size, topk, seq_len = 2 * query_len, 64, 2048, 8192
    blocks_per_req = seq_len // block_size
    logical = torch.arange(
        0, seq_len, seq_len // topk, dtype=torch.int32, device=device
    ).repeat(num_tokens, 1)
    logical[:, ::7] = -1
    builder = SparseMLAIndexGroupBuilder(logical, max_decode_rows=max_decode_rows)
    metadata = SimpleNamespace(
        block_table=torch.randperm(
            2 * blocks_per_req, dtype=torch.int32, device=device
        ).view(2, blocks_per_req),
        block_size=block_size,
        req_id_per_token=torch.arange(
            2, dtype=torch.int32, device=device
        ).repeat_interleave(query_len),
        num_decode_tokens=num_decode_tokens,
    )
    q_nope = torch.randn(num_tokens, 8, 512, dtype=torch.bfloat16, device=device)
    q_rope = torch.randn(num_tokens, 8, 64, dtype=torch.bfloat16, device=device)
    caches = torch.randn(
        num_layers,
        2 * blocks_per_req,
        block_stride_rows,
        576,
        dtype=torch.bfloat16,
        device=device,
    )[:, :, :block_size]
    impls = []
    for i in range(num_layers):
        impl = object.__new__(FlashAttnMLASparseImpl)
        impl.topk_indices_buffer = logical
        impl.index_group, impl.index_group_index = builder.register_layer(i == 0)
        impl.dcp_world_size = 1
        impl.kv_lora_rank, impl.qk_rope_head_dim = 512, 64
        impl.scale = 256**-0.5
        impl.cu_seqlens_q_buffer = torch.arange(
            num_tokens + 1, dtype=torch.int32, device=device
        )
        impls.append(impl)

    convert = triton_convert_req_index_to_global_index
    convert_spy = MagicMock(wraps=convert)
    for module in (index_group_module, sparse_mla_module):
        monkeypatch.setattr(
            module, "triton_convert_req_index_to_global_index", convert_spy
        )

    def run():
        impls[0].record_logical_topk_ready()
        return [
            impl.forward_mqa((q_nope, q_rope), cache, metadata, None)[0]
            for impl, cache in zip(impls, caches)
        ]

    stream = torch.cuda.Stream(device=device)
    stream.wait_stream(current_stream())
    with torch.cuda.stream(stream):
        run()  # Warm up the conversion and FA3 kernels before capture.
        stream.synchronize()
        convert_spy.reset_mock()
        if use_cuda_graph:
            graph = torch.cuda.CUDAGraph()
            with torch.cuda.graph(graph, stream=stream):
                outputs = run()
        else:
            outputs = run()
        reuse = num_decode_tokens == num_tokens and num_tokens <= max_decode_rows + 1
        assert convert_spy.call_count == (1 if reuse else num_layers)

        for _ in range(2):
            metadata.req_id_per_token.copy_(metadata.req_id_per_token.flip(0))
            metadata.block_table.copy_(metadata.block_table.roll(1, dims=1))
            logical.copy_(torch.where(logical >= 0, (logical + 1) % seq_len, logical))
            if use_cuda_graph:
                graph.replay()
            else:
                outputs = run()
            physical, counts = convert(
                metadata.req_id_per_token,
                metadata.block_table,
                logical,
                BLOCK_SIZE=block_size,
                BLOCK_STRIDE_ROWS=block_stride_rows,
                NUM_TOPK_TOKENS=topk,
                return_valid_counts=True,
            )
            for impl, cache, actual in zip(impls, caches, outputs):
                expected, _ = impl._forward_mqa_kernel(
                    (q_nope, q_rope),
                    cache,
                    physical,
                    counts,
                    layer=None,
                    block_size=block_size,
                    is_decode=num_decode_tokens == num_tokens,
                )
                torch.testing.assert_close(actual, expected, rtol=0, atol=0)
    stream.synchronize()


def test_flashinfer_sm120_hisparse_decode_uses_index_group():
    num_tokens = 4
    q = torch.empty(num_tokens, 2, 4, device=DEVICE_TYPE)
    topk = torch.zeros(num_tokens, 4, dtype=torch.int32, device=DEVICE_TYPE)
    physical = topk.clone()
    index_group = object.__new__(HiSparseMLAIndexGroup)
    index_group.convert_logical_to_physical_topk = MagicMock(return_value=physical)
    index_group.physical_kv_cache = MagicMock(
        return_value=torch.empty(1, device=DEVICE_TYPE)
    )
    impl = object.__new__(FlashInferMLASparseSM120Impl)
    impl.topk_indices_buffer = topk
    impl.index_group = index_group
    impl.index_group_index = 0
    impl._run_mqa_kernel = MagicMock(return_value=q[..., :1])
    metadata = SimpleNamespace(
        num_decode_tokens=num_tokens,
        topk_tokens=topk.shape[1],
    )

    output, lse = FlashInferMLASparseSM120Impl.forward_mqa(
        impl,
        q,
        torch.empty(1, device=DEVICE_TYPE),
        metadata,
        SimpleNamespace(),
    )

    assert output.shape == (num_tokens, 2, 1)
    assert lse is None
    index_group.convert_logical_to_physical_topk.assert_called_once()
    impl._run_mqa_kernel.assert_called_once()


def test_sparse_mqa_plain_lane_wiring(monkeypatch):
    """Under DCP the plain lane converts with the DCP filter at the flat stride."""
    import vllm.model_executor.layers.attention.sparse_mla_attention as sparse_mla

    num_tokens, topk_tokens, block_size, head_dim = 3, 8, 4, 6
    # Flat row stride (8) differs from block size (4): the wrong one misaddresses.
    kv_cache = torch.zeros(2, 8, head_dim)[:, :block_size]
    q = torch.zeros(num_tokens, 2, head_dim)
    metadata = SimpleNamespace(
        num_decode_tokens=num_tokens,
        block_size=block_size,
        block_table=torch.zeros((num_tokens, 1), dtype=torch.int32),
        req_id_per_token=torch.zeros(num_tokens, dtype=torch.int32),
        cp_kv_cache_interleave_size=2,
    )
    index_filter = MagicMock(return_value=(None, None))
    monkeypatch.setattr(sparse_mla, "triton_filter_and_convert_dcp_index", index_filter)

    impl = object.__new__(FlashInferMLASparseImpl)
    impl.topk_indices_buffer = torch.zeros(num_tokens, topk_tokens, dtype=torch.int32)
    impl.index_group = MagicMock()
    impl.index_group_index = 0
    impl.dcp_world_size = 2
    impl.dcp_rank = 1
    impl._convert_logical_to_physical_topk = MagicMock()
    impl._forward_mqa_kernel = MagicMock(return_value=(q, None))

    impl.forward_mqa(q, kv_cache, metadata, None)

    impl._convert_logical_to_physical_topk.assert_not_called()
    assert index_filter.call_args.kwargs["BLOCK_STRIDE_ROWS"] == 8
    impl._forward_mqa_kernel.assert_called_once()


@pytest.mark.parametrize(
    "num_decode_tokens,all_resident,split_q",
    [
        pytest.param(0, True, False, id="resident"),
        pytest.param(0, False, False, id="staged"),
        pytest.param(2, True, True, id="mixed_resident"),
        pytest.param(2, False, True, id="mixed_staged"),
        pytest.param(5, True, False, id="decode"),
    ],
)
def test_sparse_mqa_hisparse_lanes(
    monkeypatch, num_decode_tokens, all_resident, split_q
):
    """HiSparse runs all-resident and decode-only batches in one call against the
    hot cache, and otherwise splits into one decode and one staged-prefill call."""
    import vllm.model_executor.layers.attention.sparse_mla_attention as sparse_mla

    num_tokens, topk_tokens, block_size, head_dim = 5, 8, 4, 6
    num_prefill = num_tokens - num_decode_tokens
    # Distinct block counts, so the KV tensor a call is handed names its lane.
    host = torch.zeros(2, block_size, head_dim, dtype=torch.bfloat16)
    hot = torch.zeros(6, block_size, head_dim, dtype=torch.bfloat16)
    staged = torch.zeros(3, block_size, head_dim, dtype=torch.bfloat16)
    q = torch.arange(num_tokens, dtype=torch.float32).view(num_tokens, 1, 1)
    metadata = SimpleNamespace(
        num_decode_tokens=num_decode_tokens, block_size=block_size
    )

    def converted(rows, fill):
        return (
            torch.full((rows, topk_tokens), fill, dtype=torch.int32),
            torch.full((rows,), rows, dtype=torch.int32),
        )

    index_group = object.__new__(HiSparseMLAIndexGroup)
    index_group.physical_kv_cache = MagicMock(return_value=hot)
    index_group.cache = MagicMock(
        return_value=SimpleNamespace(all_context_pages_resident=all_resident)
    )
    # The fill value names the converter that produced a call's indices.
    index_group.convert_logical_to_physical_topk = MagicMock(
        side_effect=lambda layer, topk, *args, **kwargs: converted(
            topk.shape[0], 1 if topk.shape[0] == num_decode_tokens else 2
        )
    )
    index_group.stage_prefill_rows = MagicMock(return_value=(staged, None, None))
    calls: list[tuple] = []

    def convert_staged(*args, **kwargs):
        # Decode rows are consumed before a later conversion can reuse their buffer.
        assert len(calls) == min(num_decode_tokens, 1)
        return converted(num_prefill, 3)

    monkeypatch.setattr(
        sparse_mla,
        "triton_convert_req_index_to_global_index",
        MagicMock(side_effect=convert_staged),
    )

    def hook(rows, cache, indices, counts, *, is_decode, **kwargs):
        if split_q:
            assert rows[0].tolist() == rows[1].tolist()
            rows = rows[0]
        calls.append((rows.flatten().tolist(), cache.shape[0], indices[0, 0].item()))
        return torch.full((rows.shape[0], 1, 1), 1.0 if is_decode else 2.0), None

    impl = object.__new__(FlashInferMLASparseImpl)
    impl.topk_indices_buffer = torch.zeros(num_tokens, topk_tokens, dtype=torch.int32)
    impl.index_group = index_group
    impl.index_group_index = 0
    impl.dcp_world_size = 1
    impl._forward_mqa_kernel = hook

    out, lse = impl.forward_mqa((q, q) if split_q else q, host, metadata, None)

    q_rows = q.flatten().tolist()
    expected = []
    if not num_prefill:
        expected.append((q_rows, hot.shape[0], 1))
    elif all_resident:
        # Decode and prefill rows read the hot cache in one call.
        expected.append((q_rows, hot.shape[0], 2))
    else:
        if num_decode_tokens:
            expected.append((q_rows[:num_decode_tokens], hot.shape[0], 1))
        expected.append((q_rows[num_decode_tokens:], staged.shape[0], 3))
    assert calls == expected
    if not num_prefill:
        # A decode-only batch returns before it looks up the prefill cache.
        index_group.cache.assert_not_called()
    assert lse is None
    num_decode_rows = num_tokens if not num_prefill else 0
    if num_prefill and not all_resident:
        num_decode_rows = num_decode_tokens
    assert out.flatten().tolist() == [1.0] * num_decode_rows + [2.0] * (
        num_tokens - num_decode_rows
    )


@pytest.mark.parametrize("max_swap_rows", [1, 4], ids=["over_capacity", "mixed"])
def test_hisparse_resident_prefill_uses_attention_block_stride(max_swap_rows):
    """Resident batches with prefill rows read resident pages directly, whether
    they exceed the swap-row capacity or fit it."""
    expected = torch.tensor([[19]], dtype=torch.int32)
    cache_handle = SimpleNamespace(
        all_context_pages_resident=True,
        num_decode_tokens=1,
        num_actual_tokens=2,
        view=SimpleNamespace(block_size=64, attention_block_stride=832),
        block_table=torch.tensor([[3]], dtype=torch.int32),
        batch_block_table=lambda: torch.tensor([[3]], dtype=torch.int32),
        runtime=SimpleNamespace(max_swap_rows=max_swap_rows),
    )
    index_group = object.__new__(HiSparseMLAIndexGroup)
    index_group.caches = [cache_handle]
    index_group.physical_topk_indices = torch.empty((2, 1), dtype=torch.int32)
    index_group._convert_once = MagicMock(return_value=expected)
    topk = torch.zeros((2, 1), dtype=torch.int32)
    metadata = SimpleNamespace(
        req_id_per_token=torch.zeros(2, dtype=torch.int32),
    )

    result = index_group.convert_logical_to_physical_topk(
        0,
        topk,
        metadata,
        block_stride_rows=None,
        return_valid_counts=False,
    )

    assert result is expected
    assert index_group._convert_once.call_args.kwargs["block_stride_rows"] == 832


def test_hisparse_flashmla_reorders_full_speculative_window_as_decode():
    builder = object.__new__(FlashMLASparseMetadataBuilder)
    builder.vllm_config = SimpleNamespace(
        attention_config=SimpleNamespace(hisparse_config=object()),
        speculative_config=SimpleNamespace(
            num_speculative_tokens=3,
            parallel_drafting=False,
        ),
        parallel_config=SimpleNamespace(decode_context_parallel_size=1),
    )

    builder._init_reorder_batch_threshold(
        256,
        supports_spec_as_decode=True,
        supports_dcp_with_varlen=True,
    )

    assert builder.reorder_batch_threshold == 4


def test_hisparse_mixed_single_token_batch_is_not_decode_only():
    cache = SimpleNamespace(
        runtime=SimpleNamespace(
            begin_forward=lambda: None,
            eager_host_mirror=True,
        ),
    )
    metadata = SimpleNamespace(
        num_decode_tokens=1,
        num_actual_tokens=2,
        max_query_len=1,
        num_reqs=2,
        req_id_per_token=torch.arange(2),
    )

    HiSparseCacheHandle._prepare_for_batch(cache, metadata)

    assert not cache.decode_batch


def _fa4_impl(topk_indices_buffer, **fields):
    impl = object.__new__(FlashAttnMLASparseFA4Impl)
    impl.__dict__.update(
        kv_lora_rank=512,
        qk_nope_head_dim=128,
        qk_rope_head_dim=64,
        scale=1.0,
        kv_cache_dtype="auto",
        dcp_world_size=1,
        dcp_rank=0,
        need_to_return_lse_for_decode=False,
        is_nope_mla=False,
        _workspace_buffer=None,
        bmm1_scale=None,
        bmm2_scale=None,
        # A non-index-producing layer does not wait on the indexer's ready event.
        index_group=SparseMLAIndexGroupBuilder(topk_indices_buffer).register_layer(
            False
        )[0],
        index_group_index=0,
    )
    impl.__dict__.update(fields)
    # topk_indices_buffer is a data descriptor, so it needs its own setter.
    impl.topk_indices_buffer = topk_indices_buffer
    return impl


def _fa4_inputs(counts, *, num_heads=16, topk=128, num_blocks=4, device=DEVICE_TYPE):
    """Decode inputs whose token ``i`` has ``counts[i]`` valid rows, -1 padded."""
    device = torch.device(device)
    block_size, kv_lora_rank, rope_dim = 64, 512, 64
    num_tokens = len(counts)
    torch.manual_seed(0)

    def values(*shape):
        return torch.rand(shape, dtype=torch.bfloat16, device=device) - 0.5

    topk_indices = torch.full((num_tokens, topk), -1, dtype=torch.int32, device=device)
    for tok, count in enumerate(counts):
        topk_indices[tok, :count] = torch.randperm(
            num_blocks * block_size, device=device
        )[:count].to(torch.int32)

    return SimpleNamespace(
        kv_lora_rank=kv_lora_rank,
        rope_dim=rope_dim,
        block_size=block_size,
        topk=topk,
        kv_cache=values(num_blocks, block_size, kv_lora_rank + rope_dim),
        topk_indices=topk_indices,
        valid_counts=torch.tensor(counts, dtype=torch.int32, device=device),
        ql_nope=values(num_tokens, num_heads, kv_lora_rank),
        q_pe=values(num_tokens, num_heads, rope_dim),
        scale=(kv_lora_rank + rope_dim) ** -0.5,
        metadata=SimpleNamespace(
            req_id_per_token=torch.zeros(num_tokens, dtype=torch.int32, device=device),
            block_table=torch.arange(num_blocks, dtype=torch.int32, device=device).view(
                1, num_blocks
            ),
            block_size=block_size,
            num_decode_tokens=num_tokens,
            cp_kv_cache_interleave_size=1,
        ),
    )


def _record_fa4_kernels(monkeypatch, num_heads, *, device):
    """Replace both lanes with recorders whose outputs identify the lane."""
    import flashinfer.decode

    fa4_calls: list[dict] = []
    trtllm_calls: list[dict] = []

    def lse(rows):
        return torch.arange(rows * num_heads, dtype=torch.float32, device=device).view(
            rows, num_heads
        )

    def fake_fa4(**kwargs):
        fa4_calls.append(kwargs)
        rows = kwargs["q"].shape[0]
        out = torch.ones(rows, num_heads, 512, dtype=torch.bfloat16, device=device)
        return (out, lse(rows)) if kwargs.get("return_softmax_lse") else out

    def fake_trtllm(**kwargs):
        trtllm_calls.append(kwargs)
        rows = kwargs["query"].shape[0]
        out = torch.full(
            (rows, 1, num_heads, 512), 2.0, dtype=torch.bfloat16, device=device
        )
        return (out, lse(rows)) if kwargs.get("return_lse") else out

    monkeypatch.setattr(flashattn_sparse_module, "flash_attn_varlen_func", fake_fa4)
    monkeypatch.setattr(
        flashinfer.decode, "trtllm_batch_decode_with_kv_cache_mla", fake_trtllm
    )
    return SimpleNamespace(fa4=fa4_calls, trtllm=trtllm_calls, lse=lse)


@pytest.mark.parametrize(
    "num_heads,return_lse", [(128, False), (16, True)], ids=["heads128", "heads16_lse"]
)
def test_fa4_sparse_decode_kernel_correctness(num_heads, return_lse):
    if not current_platform.is_device_capability_family(100):
        pytest.skip("FA4 sparse MLA requires SM 10.x")
    if (reason := _fa4_cute_mla_available()) is not None:
        pytest.skip(reason)
    counts = [0, 1, 127, 128, 129, 2047, 2048]
    inputs = _fa4_inputs(counts, num_heads=num_heads, topk=2048, num_blocks=64)
    impl = _fa4_impl(
        topk_indices_buffer=inputs.topk_indices,
        scale=inputs.scale,
        need_to_return_lse_for_decode=return_lse,
    )
    q = (inputs.ql_nope, inputs.q_pe)

    with torch.inference_mode():
        out, lse = impl.forward_mqa(q, inputs.kv_cache, inputs.metadata, None)
        if not return_lse:
            assert lse is None
        else:
            out_fused, lse_fused = impl.forward_mqa(
                torch.cat(q, dim=-1), inputs.kv_cache, inputs.metadata, None
            )
            assert lse.shape == (len(counts), num_heads)
            assert torch.equal(out, out_fused) and torch.equal(lse, lse_fused)

    kv_flat = inputs.kv_cache.view(-1, inputs.kv_lora_rank + inputs.rope_dim)
    keys = kv_flat[inputs.topk_indices.clamp(min=0).long()].float()
    scores = torch.einsum("thd,tkd->thk", torch.cat(q, dim=-1).float(), keys)
    scores = (scores * inputs.scale).masked_fill(
        (inputs.topk_indices < 0)[:, None, :], float("-inf")
    )
    # The all-sentinel row softmaxes to NaN here; the kernel writes (0, -inf).
    probs = torch.softmax(scores, dim=-1).nan_to_num()
    torch.testing.assert_close(
        out.float(),
        torch.einsum("thk,tkd->thd", probs, keys[..., : inputs.kv_lora_rank]),
        rtol=0.01,
        atol=0.01,
    )
    if return_lse:
        torch.testing.assert_close(
            lse, torch.logsumexp(scores, dim=-1), rtol=0, atol=5e-3
        )


@pytest.mark.parametrize("is_decode", [True, False], ids=["decode", "prefill"])
def test_fa4_sparse_hook_routes_lanes(monkeypatch, is_decode):
    """The hook runs a decode batch on FA4 and every other batch on trtllm-gen."""
    import vllm.model_executor.layers.attention.sparse_mla_attention as sparse_mla

    counts = [1, 2, 3, 4, 5, 6, 7]
    num_tokens, dcp = len(counts), 2
    # Under DCP the hook gets the heads of every DCP rank, not just its own.
    kernel_heads = 16 * dcp
    inputs = _fa4_inputs(counts, num_heads=kernel_heads, device="cpu")
    calls = _record_fa4_kernels(monkeypatch, kernel_heads, device="cpu")
    # DCP without PCP (the gate rejects the pair) always fuses the query.
    q = torch.cat((inputs.ql_nope, inputs.q_pe), dim=-1)
    impl = _fa4_impl(
        # One row longer than the batch: the varlen-scalar slices become observable.
        topk_indices_buffer=inputs.topk_indices.new_zeros(num_tokens + 1, inputs.topk),
        dcp_world_size=dcp,
        need_to_return_lse_for_decode=True,
    )
    impl._convert_logical_to_physical_topk = MagicMock()
    dcp_filter = MagicMock(return_value=(inputs.topk_indices, inputs.valid_counts))
    monkeypatch.setattr(sparse_mla, "triton_filter_and_convert_dcp_index", dcp_filter)
    inputs.metadata.num_decode_tokens = num_tokens if is_decode else 1

    with torch.inference_mode():
        out, lse = impl.forward_mqa(q, inputs.kv_cache, inputs.metadata, None)

    dcp_filter.assert_called_once()
    impl._convert_logical_to_physical_topk.assert_not_called()
    assert out.shape == (num_tokens, kernel_heads, 512)
    if is_decode:
        (fa4,) = calls.fa4
        assert calls.trtllm == []
        assert fa4["return_softmax_lse"] is True
        assert fa4["cu_seqlens_q"].tolist() == list(range(num_tokens + 1))
        assert (out == 1.0).all()
        # FA4's LSE is already natural-log.
        torch.testing.assert_close(lse, calls.lse(num_tokens), rtol=0, atol=0)
    else:
        (trtllm,) = calls.trtllm
        assert calls.fa4 == []
        assert trtllm["return_lse"] is True
        assert (out == 2.0).all()
        # trtllm-gen's LSE is base 2; the DCP merge expects natural log.
        expected = calls.lse(num_tokens) * math.log(2.0)
        torch.testing.assert_close(lse, expected, rtol=0, atol=1e-5)


@pytest.mark.parametrize(
    "config,expected",
    [
        ({}, None),
        ({"heads": 128}, None),
        ({"heads": 12}, "heads"),
        ({"dcp": 2}, None),
        ({"dcp": 8}, "heads"),
        ({"index_topk": 192}, "index_topk"),
        ({"rope": 32}, "(kv_lora_rank, qk_rope_head_dim)"),
        ({"flashinfer": False}, "FlashInfer"),
    ],
)
def test_fa4_sparse_gate(monkeypatch, config, expected):
    """``validate_configuration`` on a fixed SM100: one row per gate reason."""
    has_flashinfer = config.get("flashinfer", True)
    monkeypatch.setattr("vllm.utils.flashinfer.has_flashinfer", lambda: has_flashinfer)
    monkeypatch.setattr(
        flashattn_sparse_module, "_fa4_cute_mla_available", lambda: None
    )
    dcp = config.get("dcp", 1)
    vllm_config = _build_sparse_dcp_vllm_config(config.get("heads", 16), dcp)
    hf_text_config = vllm_config.model_config.hf_text_config
    hf_text_config.index_topk = config.get("index_topk", 128)
    hf_text_config.qk_rope_head_dim = config.get("rope", 64)

    with set_current_vllm_config(vllm_config):
        reasons = FlashAttnMLASparseFA4Backend.validate_configuration(
            head_size=576,
            dtype=torch.bfloat16,
            kv_cache_dtype="auto",
            block_size=64,
            use_mla=True,
            has_sink=False,
            use_sparse=True,
            use_mm_prefix=False,
            use_per_head_quant_scales=False,
            device_capability=DeviceCapability(10, 0),
            attn_type="decoder",
            use_dcp=dcp > 1,
        )

    if expected is None:
        assert reasons == []
    else:
        (reason,) = reasons
        assert expected in reason


def test_flashmla_cache_dtype_aliases_use_ds_layout():
    # kv-cache dtype aliases are canonicalized to fp8_ds_mla before the layer
    # stores kv_cache_dtype, so they cannot bypass the gate.
    for alias in ("fp8", "fp8_e4m3"):
        assert (
            _canonicalize_sparse_mla_kv_cache_dtype(FlashMLASparseBackend, alias)
            == "fp8_ds_mla"
        )


def test_flashmla_common_metadata_requires_uniform_decodes():
    common_metadata = SimpleNamespace(
        max_query_len=3,
        num_reqs=3,
        num_actual_tokens=6,
        query_start_loc_cpu=torch.tensor([0, 1, 3, 6]),
        is_prefilling=None,
    )

    split = split_decodes_and_prefills(
        common_metadata,
        decode_threshold=128,
        require_uniform=FlashMLASparseMetadataBuilder.require_uniform_decodes,
    )

    assert split == (1, 2, 1, 5)


def test_flashmla_fp8_metadata_excludes_zero_token_decode_padding(monkeypatch):
    monkeypatch.setattr(
        "vllm.v1.attention.backends.mla.flashmla_sparse.get_mla_metadata",
        lambda: (object(), None),
    )
    builder = SimpleNamespace(
        device=torch.device(DEVICE_TYPE),
        dummy_block_table=torch.zeros(7, 1, device=DEVICE_TYPE),
        max_model_len_tensor=torch.zeros(7, device=DEVICE_TYPE),
        pcp_dcp_kv_gather=False,
    )
    query_start_loc_cpu = torch.tensor([0, 110, 220, 330, 440, 550, 660, 660])
    common_metadata = SimpleNamespace(
        num_actual_tokens=660,
        query_start_loc_cpu=query_start_loc_cpu,
        seq_lens=torch.arange(7, device=DEVICE_TYPE),
    )
    metadata = FlashMLASparseMetadata(
        num_reqs=7,
        max_query_len=110,
        max_seq_len=110,
        num_actual_tokens=660,
        query_start_loc=query_start_loc_cpu.to(DEVICE_TYPE),
        slot_mapping=torch.arange(660, device=DEVICE_TYPE),
        block_table=torch.zeros(7, 1, dtype=torch.int32, device=DEVICE_TYPE),
        req_id_per_token=torch.zeros(660, dtype=torch.int32, device=DEVICE_TYPE),
        num_decodes=7,
        num_prefills=0,
        num_decode_tokens=660,
    )

    fp8_metadata = FlashMLASparseMetadataBuilder._build_fp8_separate_prefill_decode(
        builder, common_metadata, metadata
    )

    assert fp8_metadata.num_decodes == 6
    assert fp8_metadata.num_decode_tokens == 660
    assert fp8_metadata.decode is not None
    assert fp8_metadata.decode.decode_query_len == 110
    torch.testing.assert_close(
        fp8_metadata.decode.seq_lens, torch.arange(6, device=DEVICE_TYPE)
    )


@pytest.mark.parametrize("use_mixed_batch", [False, True])
def test_flashmla_fp8_paths_accept_decode_subset(monkeypatch, use_mixed_batch: bool):
    num_decode_tokens = 2
    num_batch_tokens = 5
    q = torch.empty(num_decode_tokens, 2, 3, device=DEVICE_TYPE)
    topk_indices = torch.empty(num_decode_tokens, 4, device=DEVICE_TYPE)
    kernel_q_shapes = []

    def convert_indices(*args, **kwargs):  # noqa: ARG001
        assert not kwargs.get("HAS_PREFILL_WORKSPACE", False)
        if not kwargs.get("return_valid_counts", False):
            return topk_indices
        valid_counts = torch.full(
            (num_decode_tokens,), 4, dtype=torch.int32, device=DEVICE_TYPE
        )
        return topk_indices, valid_counts

    monkeypatch.setattr(
        "vllm.v1.attention.backends.mla.flashmla_sparse."
        "triton_convert_req_index_to_global_index",
        convert_indices,
    )

    def run_kernel(**kwargs):
        kernel_q_shapes.append(kwargs["q"].shape)
        return kwargs["q"][..., :1], None

    if use_mixed_batch:
        fp8_metadata = FlashMLASparseMetadata.FP8KernelMetadata(
            scheduler_metadata=object(),
            dummy_block_table=torch.empty(1, 1, dtype=torch.int32, device=DEVICE_TYPE),
            cache_lens=torch.empty(1, dtype=torch.int32, device=DEVICE_TYPE),
        )
    else:
        FP8Meta = FlashMLASparseMetadata.FP8SeparatePrefillDecode
        fp8_metadata = FP8Meta(
            num_decodes=1,
            num_prefills=1,
            num_decode_tokens=num_decode_tokens,
            num_prefill_tokens=num_batch_tokens - num_decode_tokens,
            decode=FP8Meta.Decode(
                seq_lens=torch.empty(1, dtype=torch.int32, device=DEVICE_TYPE),
                kernel_metadata=object(),
                decode_query_len=num_decode_tokens,
            ),
            prefill=FP8Meta.Prefill(
                request_ids=torch.empty(
                    num_batch_tokens, dtype=torch.int32, device=DEVICE_TYPE
                ),
                workspace_starts=torch.empty(1, dtype=torch.int32, device=DEVICE_TYPE),
                chunks=[],
            ),
        )
    metadata = SimpleNamespace(
        fp8_extra_metadata=fp8_metadata,
        fp8_use_mixed_batch=use_mixed_batch,
        num_actual_tokens=num_batch_tokens,
        num_decode_tokens=num_decode_tokens,
        req_id_per_token=torch.empty(
            num_batch_tokens, dtype=torch.int32, device=DEVICE_TYPE
        ),
        block_table=torch.empty(1, 1, dtype=torch.int32, device=DEVICE_TYPE),
        block_size=64,
    )
    impl = SimpleNamespace(
        kv_cache_dtype="fp8_ds_mla",
        topk_indices_buffer=topk_indices,
        num_heads=2,
        kv_lora_rank=1,
        q_head_size=3,
        index_group=None,
        index_group_index=0,
        dcp_world_size=1,
        pcp_dcp_kv_gather=False,
        need_to_return_lse_for_decode=False,
        _fp8_flash_mla_kernel=run_kernel,
        _convert_logical_to_physical_topk=(
            lambda indices, metadata, **kwargs: convert_indices(
                metadata.req_id_per_token[: indices.shape[0]],
                metadata.block_table,
                indices,
                return_valid_counts=kwargs["return_valid_counts"],
            )
        ),
    )
    impl._forward_fp8_kv_mixed_batch = MethodType(
        FlashMLASparseImpl._forward_fp8_kv_mixed_batch, impl
    )
    impl._forward_fp8_kv_separate_prefill_decode = MethodType(
        FlashMLASparseImpl._forward_fp8_kv_separate_prefill_decode, impl
    )

    output, lse = FlashMLASparseImpl.forward_mqa(
        impl,
        q,
        torch.empty(0, device=DEVICE_TYPE),
        metadata,
        None,
    )

    assert kernel_q_shapes == [(1, num_decode_tokens, 2, 3)]
    assert output.shape == (num_decode_tokens, 2, 1)
    assert lse is None


def _build_sparse_dcp_vllm_config(
    local_heads: int,
    dcp_world_size: int,
    comm_backend: str = "ag_rs",
):
    """Minimal sparse-MLA VllmConfig for the FlashMLASparse DCP head-envelope
    guard. TP is simulated by mocking ``get_num_attention_heads`` to return the
    per-rank head count, as the decode-correctness test above does.
    """
    kv_lora_rank = 512
    qk_nope_head_dim = 128
    qk_rope_head_dim = 64
    v_head_dim = 128
    head_size = kv_lora_rank + qk_rope_head_dim
    topk_tokens = 128

    vllm_config = create_vllm_config(
        model_name="deepseek-ai/DeepSeek-V2-Lite-Chat",
        tensor_parallel_size=1,
        max_model_len=4096,
        block_size=64,
        hf_config_override={
            "index_topk": topk_tokens,
            "attn_module_list_cfg": [{"topk_tokens": topk_tokens}],
        },
    )
    model_config = vllm_config.model_config
    model_config.dtype = torch.bfloat16
    model_config.hf_text_config = SimpleNamespace(
        index_topk=topk_tokens,
        q_lora_rank=None,
        kv_lora_rank=kv_lora_rank,
        qk_nope_head_dim=qk_nope_head_dim,
        qk_rope_head_dim=qk_rope_head_dim,
        v_head_dim=v_head_dim,
        model_type="deepseek_v2",
    )
    del model_config.hf_config.index_topk  # Composite configs only nest this field.
    model_config.get_num_attention_heads = MethodType(
        lambda self, parallel_config: local_heads, model_config
    )
    model_config.get_num_kv_heads = MethodType(
        lambda self, parallel_config: 1, model_config
    )
    model_config.get_head_size = MethodType(lambda self: head_size, model_config)
    model_config.get_sliding_window = MethodType(lambda self: None, model_config)

    vllm_config.cache_config.cache_dtype = "fp8_ds_mla"
    vllm_config.parallel_config.decode_context_parallel_size = dcp_world_size
    vllm_config.parallel_config.dcp_comm_backend = comm_backend
    # The base builder clones the layer's dense-MHA prefill backend from
    # static_forward_context; the guard tests never run prefill.
    vllm_config.compilation_config.static_forward_context["placeholder"] = (
        SimpleNamespace(prefill_backend=None)
    )
    return vllm_config


@pytest.mark.skipif(
    torch.cuda.get_device_capability() < (9, 0),
    reason="FlashMLASparseBackend requires CUDA 9.0 or higher",
)
@pytest.mark.parametrize(
    "local_heads,dcp_world_size,should_raise",
    [
        (16, 8, True),
        (24, 4, True),
        (16, 4, False),
        (16, 1, False),
    ],
)
def test_fp8_dcp_head_envelope_guard(local_heads, dcp_world_size, should_raise):
    """The fp8 decode envelope (head padding + tile-scheduler metadata) is
    sized from the local head count while the kernel runs on the DCP-gathered
    heads, so the builder must reject configs where the two pad differently.
    """
    device = torch.device(DEVICE_TYPE)
    vllm_config = _build_sparse_dcp_vllm_config(local_heads, dcp_world_size)
    kv_cache_spec = create_standard_kv_cache_spec(vllm_config)
    builder_cls = FlashMLASparseBackend.get_builder_cls()

    if should_raise:
        with pytest.raises(NotImplementedError, match="envelope"):
            builder_cls(kv_cache_spec, ["placeholder"], vllm_config, device)
    else:
        builder = builder_cls(kv_cache_spec, ["placeholder"], vllm_config, device)
        gathered_heads = local_heads * dcp_world_size
        local_pad = 64 if local_heads <= 64 else 128
        gathered_pad = 64 if gathered_heads <= 64 else 128
        assert builder.fp8_decode_padded_heads == local_pad
        assert local_pad == gathered_pad


@pytest.mark.skipif(
    torch.cuda.get_device_capability() < (9, 0),
    reason="FlashMLASparseBackend requires CUDA 9.0 or higher",
)
@pytest.mark.parametrize("comm_backend", ["ag_rs", "a2a"])
def test_fp8_dcp_accepts_comm_backend(comm_backend):
    """The mixed-batch path returns (out, lse) for every token in the layout
    both DCP combines consume, so neither comm backend is rejected."""
    device = torch.device(DEVICE_TYPE)
    vllm_config = _build_sparse_dcp_vllm_config(16, 4, comm_backend)
    kv_cache_spec = create_standard_kv_cache_spec(vllm_config)
    builder_cls = FlashMLASparseBackend.get_builder_cls()
    builder = builder_cls(kv_cache_spec, ["placeholder"], vllm_config, device)
    assert builder.fp8_use_mixed_batch


def test_fp8_mixed_batch_dcp_neutralizes_empty_rows(monkeypatch):
    """A decode row whose top-k shard holds no local candidates (all -1) has
    undefined kernel out/lse; it must come back as (0, -inf), the identity of
    the cross-rank LSE merge, or a NaN would survive the merge even at zero
    weight (0 * NaN = NaN)."""
    num_tokens, num_heads, head_dim = 3, 2, 3
    q = torch.empty(num_tokens, num_heads, head_dim, device=DEVICE_TYPE)
    local_indices = torch.tensor(
        [[0, 1, -1, -1], [-1, -1, -1, -1], [2, -1, 3, -1]],
        dtype=torch.int32,
        device=DEVICE_TYPE,
    )

    monkeypatch.setattr(
        "vllm.v1.attention.backends.mla.flashmla_sparse."
        "triton_filter_and_convert_dcp_index",
        lambda *args, **kwargs: local_indices,
    )

    def run_kernel(**kwargs):
        out = torch.full(
            (1, num_tokens, num_heads, 1), float("nan"), device=DEVICE_TYPE
        )
        lse = torch.full((1, num_heads, num_tokens), float("nan"), device=DEVICE_TYPE)
        for token_id in (0, 2):  # rows with local candidates get real values
            out[0, token_id] = float(token_id + 1)
            lse[0, :, token_id] = float(token_id + 1)
        return out, lse

    metadata = SimpleNamespace(
        fp8_extra_metadata=FlashMLASparseMetadata.FP8KernelMetadata(
            scheduler_metadata=object(),
            dummy_block_table=torch.empty(1, 1, dtype=torch.int32, device=DEVICE_TYPE),
            cache_lens=torch.empty(1, dtype=torch.int32, device=DEVICE_TYPE),
        ),
        req_id_per_token=torch.empty(num_tokens, dtype=torch.int32, device=DEVICE_TYPE),
        block_table=torch.empty(1, 1, dtype=torch.int32, device=DEVICE_TYPE),
        block_size=64,
        cp_kv_cache_interleave_size=1,
    )
    impl = SimpleNamespace(
        index_group=None,
        index_group_index=0,
        dcp_world_size=2,
        dcp_rank=0,
        need_to_return_lse_for_decode=True,
        _fp8_flash_mla_kernel=run_kernel,
    )

    out, lse = FlashMLASparseImpl._forward_fp8_kv_mixed_batch(
        impl, q, torch.empty(0, device=DEVICE_TYPE), local_indices, metadata
    )

    assert torch.equal(out[1], torch.zeros_like(out[1]))
    assert torch.isneginf(lse[1]).all()
    for token_id in (0, 2):
        assert torch.equal(out[token_id], torch.full_like(out[token_id], token_id + 1))
        assert torch.equal(lse[token_id], torch.full_like(lse[token_id], token_id + 1))
    assert out.is_contiguous()
    assert not out.isnan().any()
    assert not lse.isnan().any()


def test_hisparse_prefill_reuses_builder_staging_plan():
    """Every layer must reuse the batch plan instead of synchronizing to dedupe."""
    plan = SimpleNamespace(
        block_table=torch.tensor([[0]], dtype=torch.int32),
        ensure_gpu_sources=MagicMock(),
    )
    staged = torch.empty((1, 1, 8))
    calls = []

    def gather(kv_cache, staging_plan, resident_cache=None):
        calls.append((kv_cache, staging_plan, resident_cache))
        return staged

    resident_cache = torch.empty((1, 1, 8))
    resident_block_table = torch.tensor([[1]], dtype=torch.int32)
    cache = SimpleNamespace(
        runtime=SimpleNamespace(
            gather_prefill_cache=gather,
            request_state_indices=torch.tensor([0], dtype=torch.int32),
        ),
        view=SimpleNamespace(cache=resident_cache, block_size=1),
        block_table=resident_block_table,
    )
    index_group = object.__new__(HiSparseMLAIndexGroup)
    index_group.caches = [cache]
    source = torch.empty((1, 1, 8))
    metadata = SimpleNamespace(
        num_decodes=0,
        num_decode_tokens=0,
        seq_lens=torch.tensor([1], dtype=torch.int32),
        prefill=SimpleNamespace(host_staging_plan=plan),
        req_id_per_token=torch.tensor([0], dtype=torch.int32),
    )

    result, block_table, request_ids = index_group.stage_prefill_rows(
        0, source, metadata
    )

    assert result is staged
    assert block_table is plan.block_table
    torch.testing.assert_close(request_ids, metadata.req_id_per_token)
    assert calls == [(source, plan, resident_cache)]


def test_hisparse_fp8_prefill_gather_uses_dedicated_stream(monkeypatch):
    compute_stream = object()
    prefill_stream = MagicMock()
    gather = MagicMock()
    monkeypatch.setattr(index_group_module, "current_stream", lambda: compute_stream)
    monkeypatch.setattr(
        index_group_module.ops, "cp_gather_and_upconvert_fp8_kv_cache", gather
    )

    resident_cache = torch.empty((2, 4, 656), dtype=torch.uint8)
    ready = MagicMock()
    cache = SimpleNamespace(
        view=SimpleNamespace(cache=resident_cache, block_size=4),
        block_table=torch.tensor([[2, 3]], dtype=torch.int32),
        runtime=SimpleNamespace(
            request_state_indices=torch.tensor([0], dtype=torch.int32)
        ),
    )
    index_group = object.__new__(HiSparseMLAIndexGroup)
    index_group.caches = [cache]
    index_group.prefill_stream = prefill_stream
    index_group.prefill_ready_events = [ready]
    source = torch.empty((3, 4, 656), dtype=torch.uint8)
    dst = torch.empty((8, 576), dtype=torch.bfloat16)
    block_table = torch.tensor([[0, 1]], dtype=torch.int32)
    workspace_starts = torch.tensor([0], dtype=torch.int32)
    plan = SimpleNamespace(
        block_table=torch.tensor([[0, 1]], dtype=torch.int32),
        row_ids=torch.arange(8, dtype=torch.int32).view(1, -1),
        gpu_row_ids=torch.full((1, 8), -1, dtype=torch.int32),
        ensure_gpu_sources=MagicMock(),
    )
    metadata = SimpleNamespace(
        prefill=SimpleNamespace(host_staging_plan=plan),
        num_decodes=0,
    )

    result = index_group.gather_fp8_prefill(
        0,
        source,
        dst,
        block_table,
        workspace_starts,
        1,
        metadata,
        0,
    )

    prefill_stream.wait_stream.assert_called_once_with(compute_stream)
    ready.record.assert_called_once_with(prefill_stream)
    assert result is ready
    gather.assert_called_once()
    args = gather.call_args
    assert args.args[0] is resident_cache
    assert args.args[1] is dst
    torch.testing.assert_close(args.args[2], plan.block_table)
    assert args.args[3] is workspace_starts
    assert args.args[4] == 1
    assert args.kwargs["host_cache"].data_ptr() == source.data_ptr()
    assert args.kwargs["host_row_ids"] is plan.row_ids
    assert args.kwargs["device_row_ids"] is plan.gpu_row_ids


def test_sparse_impl_observes_repointed_indexer_buffer():
    """The MTP proposer repoints the draft's indexer at the target model's buffer
    after the backend impl is built, so the impl must resolve the buffer per read.
    Snapshotting it in __init__ leaves the layer reading indices nothing writes."""
    impl = object.__new__(FlashInferMLASparseImpl)
    own = torch.zeros(4, 8, dtype=torch.int32)
    target = torch.ones(4, 8, dtype=torch.int32)
    indexer = SimpleNamespace(topk_indices_buffer=own)
    impl.init_topk_indices_buffer(indexer, None)

    assert impl.topk_indices_buffer is own

    indexer.topk_indices_buffer = target

    assert impl.topk_indices_buffer is target


def test_explicit_topk_buffer_supersedes_indexer():
    """Backbone skip-topk layers have no indexer, and the proposer also assigns the
    shared buffer directly onto draft submodules."""
    impl = object.__new__(FlashInferMLASparseImpl)
    indexer = SimpleNamespace(
        topk_indices_buffer=torch.zeros(2, 2, dtype=torch.int32),
    )
    impl.init_topk_indices_buffer(indexer, None)
    explicit = torch.ones(2, 2, dtype=torch.int32)

    impl.topk_indices_buffer = explicit

    assert impl.topk_indices_buffer is explicit


def test_sparse_mla_common_impl_resolves_buffer_lazily():
    """Guards the whole SparseMLACommonImpl family at once: a backend that
    snapshots the buffer instead silently loses MTP buffer sharing."""
    assert issubclass(SparseMLACommonImpl, SharedTopkIndicesBuffer)
    assert isinstance(SparseMLACommonImpl.topk_indices_buffer, property), (
        "topk_indices_buffer must stay a lazily-resolved property"
    )


@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm model implementation")
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
def test_rocm_fused_v32_dense_prefill_matches_mqa(
    tmp_path, vllm_runner, kv_cache_dtype
):
    """Fused normalization, RoPE and mixed dispatch preserve generation/logprobs.

    Chunking unequal prompts exercises prefill alongside decode. The two long
    prompts straddle the dense-prefill threshold. No pretrained weights or
    tokenizer download is needed.
    """
    import json

    from vllm import SamplingParams

    config = {
        "architectures": ["DeepseekV32ForCausalLM"],
        "attention_bias": False,
        "attention_dropout": 0.0,
        "bos_token_id": 0,
        "eos_token_id": 1,
        "ep_size": 1,
        "first_k_dense_replace": 1,
        "hidden_act": "silu",
        "hidden_size": 2048,
        "index_head_dim": 128,
        "index_n_heads": 64,
        "index_topk": 2048,
        "initializer_range": 0.02,
        "intermediate_size": 4096,
        "kv_lora_rank": 512,
        "max_position_embeddings": 8192,
        "model_type": "deepseek_v32",
        "moe_intermediate_size": 512,
        "moe_layer_freq": 1,
        "n_group": 1,
        "n_routed_experts": 8,
        "n_shared_experts": 1,
        "norm_topk_prob": True,
        "num_attention_heads": 32,
        "num_experts_per_tok": 2,
        "num_hidden_layers": 2,
        "num_key_value_heads": 32,
        "num_nextn_predict_layers": 0,
        "q_lora_rank": 512,
        "qk_nope_head_dim": 128,
        "qk_rope_head_dim": 64,
        "rms_norm_eps": 1e-06,
        "rope_scaling": {
            "beta_fast": 32,
            "beta_slow": 1,
            "factor": 40,
            "mscale": 1.0,
            "mscale_all_dim": 1.0,
            "original_max_position_embeddings": 4096,
            "type": "yarn",
        },
        "rope_theta": 10000,
        "routed_scaling_factor": 2.5,
        "scoring_func": "sigmoid",
        "tie_word_embeddings": False,
        "topk_group": 1,
        "topk_method": "noaux_tc",
        "torch_dtype": "bfloat16",
        "transformers_version": "4.44.2",
        "use_cache": True,
        "v_head_dim": 128,
        "vocab_size": 4096,
    }
    (tmp_path / "config.json").write_text(json.dumps(config))
    prompts = [
        {"prompt_token_ids": [2 + (i * 17 + j * 13) % 4000 for i in range(n)]}
        for j, n in enumerate([64, 128, 2048, 2049])
    ]
    sampling = SamplingParams(temperature=0, max_tokens=16, ignore_eos=True, logprobs=5)
    results = []
    for force_mqa in [True, False]:
        with vllm_runner(
            str(tmp_path),
            load_format="dummy",
            skip_tokenizer_init=True,
            dtype="bfloat16",
            kv_cache_dtype=kv_cache_dtype,
            seed=42,
            max_model_len=4096,
            max_num_seqs=4,
            max_num_batched_tokens=256,
            block_size=64,
            enable_chunked_prefill=True,
            enable_prefix_caching=False,
            enforce_eager=True,
            gpu_memory_utilization=0.25,
            attention_config={
                "backend": "ROCM_AITER_MLA_SPARSE",
                "sparse_mla_force_mqa": force_mqa,
            },
        ) as runner:
            outputs = runner.llm.generate(prompts, sampling, use_tqdm=False)
            results.append([request.outputs[0] for request in outputs])

    for reference, actual in zip(*results):
        assert len(actual.token_ids) == 16
        assert actual.token_ids == reference.token_ids
        assert actual.logprobs is not None and reference.logprobs is not None
        for token, expected, observed in zip(
            actual.token_ids, reference.logprobs, actual.logprobs
        ):
            torch.testing.assert_close(
                torch.tensor(observed[token].logprob),
                torch.tensor(expected[token].logprob),
                rtol=0.01,
                atol=0.01,
            )