[3/N] HiSparse: host-resident sparse-MLA decode hot-buffering (#53781)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Signed-off-by: Matthew Bonanni <matthew@neuralmagic.com>
Signed-off-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
Matthew Bonanni
2026-09-12 00:36:47 -04:00
committed by GitHub
co-authored by OpenAI Codex Lucas Wilkinson Lucas Wilkinson Nicolò Lucchesi OpenAI Codex Claude Opus 5 Lucas Wilkinson
parent 120ec4ebd2
commit d43bb2f37f
124 changed files with 14262 additions and 906 deletions
@@ -217,7 +217,8 @@ def test_flashinfer_mla_dense_decode_unified_slot_view():
assert (out_ref - out_view).abs().max().item() == 0.0
def test_flashmla_fp8_sparse_decode_unified_slot_view():
@pytest.mark.parametrize("next_n", [1, 2, 4])
def test_flashmla_fp8_sparse_decode_unified_slot_view(next_n: int):
"""FlashMLA fp8 sparse decode (DeepSeek V3.2/V4 DSA path) must read a
unified-slot block-major view bit-identically to a contiguous fp8_ds_mla
cache, with finite nonzero output."""
@@ -240,7 +241,10 @@ def test_flashmla_fp8_sparse_decode_unified_slot_view():
n_layers = 3
layer = 1
q = torch.randn(batch, 1, h_q, head_dim, device=dev, dtype=torch.bfloat16) * 0.1
q = (
torch.randn(batch, next_n, h_q, head_dim, device=dev, dtype=torch.bfloat16)
* 0.1
)
# Structurally valid fp8 ds_mla payload: 512B fp8 + 16B f32 scales + 128B
# bf16 rope (random bytes corrupt the scale region and yield NaNs).
@@ -273,13 +277,13 @@ def test_flashmla_fp8_sparse_decode_unified_slot_view():
# Sparse indices: each batch uses its own disjoint blocks.
blocks_per_batch = num_blocks // batch
idx = torch.full((batch, 1, topk), -1, device=dev, dtype=torch.int32)
idx = torch.full((batch, next_n, topk), -1, device=dev, dtype=torch.int32)
for b in range(batch):
slots: list[int] = []
for blk in range(b * blocks_per_batch, (b + 1) * blocks_per_batch):
slots.extend(blk * page + off for off in range(page))
slots_t = torch.tensor(slots[:topk], device=dev, dtype=torch.int32)
idx[b, 0, : slots_t.numel()] = slots_t
idx[b, :, : slots_t.numel()] = slots_t
def run(kc):
meta, num_splits = fm.get_mla_metadata()
+42
View File
@@ -316,6 +316,48 @@ def test_cp_gather_fp8_shuffled_blocks():
assert torch.equal(dst[:, NOPE_DIM:], expected[:, NOPE_DIM:])
def test_cp_gather_fp8_mixes_resident_and_pinned_host_rows():
seq_lens = [7, 5]
block_size = 4
(
cache,
block_table,
workspace_starts,
num_reqs,
total_tokens,
expected,
) = _build_test_case(seq_lens, block_size)
device_cache = cache.clone()
host_cache = cache.cpu().pin_memory().view(-1, ENTRY_BYTES)
device_rows = torch.arange(host_cache.shape[0], dtype=torch.int32, device="cuda")
use_device = device_rows % 2 == 0
host_rows = device_rows.clone()
device_rows.masked_fill_(~use_device, -1)
host_cache[use_device.cpu()] = 0
device_cache.view(-1, ENTRY_BYTES)[~use_device] = 0
dst = torch.empty(
total_tokens, NOPE_DIM + ROPE_DIM, dtype=torch.bfloat16, device="cuda"
)
ops.cp_gather_and_upconvert_fp8_kv_cache(
device_cache,
dst,
block_table,
workspace_starts,
num_reqs,
host_cache=host_cache,
host_row_ids=host_rows,
device_row_ids=device_rows,
)
torch.testing.assert_close(
dst[:, :NOPE_DIM], expected[:, :NOPE_DIM], atol=1e-3, rtol=1e-2
)
assert torch.equal(dst[:, NOPE_DIM:], expected[:, NOPE_DIM:])
@pytest.mark.parametrize(
"gather_seq_lens,seq_starts",
[
@@ -234,6 +234,76 @@ def test_fused_norm_rope(num_tokens: int, index_interleave: bool, mla_dtype: str
assert (topk == -1).all(), "topk buffer not cleared on indexer layer"
def test_fused_norm_rope_packed_indexer_block_stride():
"""Indexer writes use their HMA block stride and independent slot map."""
torch.manual_seed(7)
dev = "cuda"
num_tokens = block_size = 2
pos = torch.arange(num_tokens, device=dev, dtype=torch.int64)
q_c = torch.randn(num_tokens, Q_LORA, device=dev, dtype=torch.bfloat16)
kv_c = torch.randn(num_tokens, KV_LORA, device=dev, dtype=torch.bfloat16)
k_pe = torch.randn(num_tokens, ROPE_DIM, device=dev, dtype=torch.bfloat16)
qw = torch.randn(Q_LORA, device=dev, dtype=torch.bfloat16)
kvw = torch.randn(KV_LORA, device=dev, dtype=torch.bfloat16)
ik = torch.randn(num_tokens, INDEX_HEAD_DIM, device=dev, dtype=torch.bfloat16)
ikw = torch.randn(INDEX_HEAD_DIM, device=dev, dtype=torch.float32)
ikb = torch.randn(INDEX_HEAD_DIM, device=dev, dtype=torch.float32)
cos_sin = make_cos_sin(32, ROPE_DIM, dev)
idx_row = INDEX_HEAD_DIM + INDEX_HEAD_DIM // 128 * 4
packed_block_stride = block_size * idx_row + 64
backing = torch.zeros(2 * packed_block_stride, device=dev, dtype=torch.uint8)
idx_cache = torch.as_strided(
backing,
(2, block_size, idx_row),
(packed_block_stride, idx_row, 1),
)
mla_cache = torch.zeros(
1,
block_size,
KV_LORA + ROPE_DIM,
device=dev,
dtype=torch.bfloat16,
)
topk = torch.zeros(num_tokens, 8, device=dev, dtype=torch.int32)
K.fused_norm_rope(
pos,
q_c,
qw,
EPS,
kv_c,
kvw,
EPS,
k_pe,
cos_sin,
ik,
ikw,
ikb,
EPS,
cos_sin,
topk,
slot_mapping=torch.arange(num_tokens, device=dev, dtype=torch.int64),
indexer_slot_mapping=torch.arange(
block_size, block_size + num_tokens, device=dev, dtype=torch.int64
),
indexer_k_cache=idx_cache,
mla_kv_cache=mla_cache,
has_indexer=True,
)
ik_ref = rope(layer_norm(ik, ikw, ikb), pos, cos_sin, interleave=False)
q_ref, s_ref = ue8m0_quant(ik_ref)
packed = idx_cache[1].reshape(-1)
values = (
packed[: block_size * INDEX_HEAD_DIM].view(FP8).view(block_size, INDEX_HEAD_DIM)
)
scales = packed[block_size * INDEX_HEAD_DIM :].view(torch.float32)
assert_fp8(values, q_ref, "packed indexer-K fp8")
torch.testing.assert_close(scales, s_ref, rtol=0, atol=0)
assert (backing[block_size * idx_row : packed_block_stride] == 0).all()
def test_fused_norm_rope_materializes_qk_without_local_cache_slots():
"""DCP non-owner ranks still need valid query shards for query AllGather."""
torch.manual_seed(7)