[3/N] HiSparse: host-resident sparse-MLA decode hot-buffering (#53781)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Signed-off-by: Matthew Bonanni <matthew@neuralmagic.com>
Signed-off-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
Matthew Bonanni
2026-09-12 00:36:47 -04:00
committed by GitHub
co-authored by OpenAI Codex Lucas Wilkinson Lucas Wilkinson Nicolò Lucchesi OpenAI Codex Claude Opus 5 Lucas Wilkinson
parent 120ec4ebd2
commit d43bb2f37f
124 changed files with 14262 additions and 906 deletions
@@ -217,7 +217,8 @@ def test_flashinfer_mla_dense_decode_unified_slot_view():
assert (out_ref - out_view).abs().max().item() == 0.0
def test_flashmla_fp8_sparse_decode_unified_slot_view():
@pytest.mark.parametrize("next_n", [1, 2, 4])
def test_flashmla_fp8_sparse_decode_unified_slot_view(next_n: int):
"""FlashMLA fp8 sparse decode (DeepSeek V3.2/V4 DSA path) must read a
unified-slot block-major view bit-identically to a contiguous fp8_ds_mla
cache, with finite nonzero output."""
@@ -240,7 +241,10 @@ def test_flashmla_fp8_sparse_decode_unified_slot_view():
n_layers = 3
layer = 1
q = torch.randn(batch, 1, h_q, head_dim, device=dev, dtype=torch.bfloat16) * 0.1
q = (
torch.randn(batch, next_n, h_q, head_dim, device=dev, dtype=torch.bfloat16)
* 0.1
)
# Structurally valid fp8 ds_mla payload: 512B fp8 + 16B f32 scales + 128B
# bf16 rope (random bytes corrupt the scale region and yield NaNs).
@@ -273,13 +277,13 @@ def test_flashmla_fp8_sparse_decode_unified_slot_view():
# Sparse indices: each batch uses its own disjoint blocks.
blocks_per_batch = num_blocks // batch
idx = torch.full((batch, 1, topk), -1, device=dev, dtype=torch.int32)
idx = torch.full((batch, next_n, topk), -1, device=dev, dtype=torch.int32)
for b in range(batch):
slots: list[int] = []
for blk in range(b * blocks_per_batch, (b + 1) * blocks_per_batch):
slots.extend(blk * page + off for off in range(page))
slots_t = torch.tensor(slots[:topk], device=dev, dtype=torch.int32)
idx[b, 0, : slots_t.numel()] = slots_t
idx[b, :, : slots_t.numel()] = slots_t
def run(kc):
meta, num_splits = fm.get_mla_metadata()