[3/N] HiSparse: host-resident sparse-MLA decode hot-buffering (#53781)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com> Signed-off-by: Matthew Bonanni <matthew@neuralmagic.com> Signed-off-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Lucas Wilkinson <lwilkinson@neuralmagic.com> Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
Lucas Wilkinson
Lucas Wilkinson
Nicolò Lucchesi
OpenAI Codex
Claude Opus 5
Lucas Wilkinson
parent
120ec4ebd2
commit
d43bb2f37f
@@ -217,7 +217,8 @@ def test_flashinfer_mla_dense_decode_unified_slot_view():
|
||||
assert (out_ref - out_view).abs().max().item() == 0.0
|
||||
|
||||
|
||||
def test_flashmla_fp8_sparse_decode_unified_slot_view():
|
||||
@pytest.mark.parametrize("next_n", [1, 2, 4])
|
||||
def test_flashmla_fp8_sparse_decode_unified_slot_view(next_n: int):
|
||||
"""FlashMLA fp8 sparse decode (DeepSeek V3.2/V4 DSA path) must read a
|
||||
unified-slot block-major view bit-identically to a contiguous fp8_ds_mla
|
||||
cache, with finite nonzero output."""
|
||||
@@ -240,7 +241,10 @@ def test_flashmla_fp8_sparse_decode_unified_slot_view():
|
||||
n_layers = 3
|
||||
layer = 1
|
||||
|
||||
q = torch.randn(batch, 1, h_q, head_dim, device=dev, dtype=torch.bfloat16) * 0.1
|
||||
q = (
|
||||
torch.randn(batch, next_n, h_q, head_dim, device=dev, dtype=torch.bfloat16)
|
||||
* 0.1
|
||||
)
|
||||
|
||||
# Structurally valid fp8 ds_mla payload: 512B fp8 + 16B f32 scales + 128B
|
||||
# bf16 rope (random bytes corrupt the scale region and yield NaNs).
|
||||
@@ -273,13 +277,13 @@ def test_flashmla_fp8_sparse_decode_unified_slot_view():
|
||||
|
||||
# Sparse indices: each batch uses its own disjoint blocks.
|
||||
blocks_per_batch = num_blocks // batch
|
||||
idx = torch.full((batch, 1, topk), -1, device=dev, dtype=torch.int32)
|
||||
idx = torch.full((batch, next_n, topk), -1, device=dev, dtype=torch.int32)
|
||||
for b in range(batch):
|
||||
slots: list[int] = []
|
||||
for blk in range(b * blocks_per_batch, (b + 1) * blocks_per_batch):
|
||||
slots.extend(blk * page + off for off in range(page))
|
||||
slots_t = torch.tensor(slots[:topk], device=dev, dtype=torch.int32)
|
||||
idx[b, 0, : slots_t.numel()] = slots_t
|
||||
idx[b, :, : slots_t.numel()] = slots_t
|
||||
|
||||
def run(kc):
|
||||
meta, num_splits = fm.get_mla_metadata()
|
||||
|
||||
Reference in New Issue
Block a user