[3/N] HiSparse: host-resident sparse-MLA decode hot-buffering (#53781)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com> Signed-off-by: Matthew Bonanni <matthew@neuralmagic.com> Signed-off-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Lucas Wilkinson <lwilkinson@neuralmagic.com> Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
Lucas Wilkinson
Lucas Wilkinson
Nicolò Lucchesi
OpenAI Codex
Claude Opus 5
Lucas Wilkinson
parent
120ec4ebd2
commit
d43bb2f37f
@@ -217,7 +217,8 @@ def test_flashinfer_mla_dense_decode_unified_slot_view():
|
||||
assert (out_ref - out_view).abs().max().item() == 0.0
|
||||
|
||||
|
||||
def test_flashmla_fp8_sparse_decode_unified_slot_view():
|
||||
@pytest.mark.parametrize("next_n", [1, 2, 4])
|
||||
def test_flashmla_fp8_sparse_decode_unified_slot_view(next_n: int):
|
||||
"""FlashMLA fp8 sparse decode (DeepSeek V3.2/V4 DSA path) must read a
|
||||
unified-slot block-major view bit-identically to a contiguous fp8_ds_mla
|
||||
cache, with finite nonzero output."""
|
||||
@@ -240,7 +241,10 @@ def test_flashmla_fp8_sparse_decode_unified_slot_view():
|
||||
n_layers = 3
|
||||
layer = 1
|
||||
|
||||
q = torch.randn(batch, 1, h_q, head_dim, device=dev, dtype=torch.bfloat16) * 0.1
|
||||
q = (
|
||||
torch.randn(batch, next_n, h_q, head_dim, device=dev, dtype=torch.bfloat16)
|
||||
* 0.1
|
||||
)
|
||||
|
||||
# Structurally valid fp8 ds_mla payload: 512B fp8 + 16B f32 scales + 128B
|
||||
# bf16 rope (random bytes corrupt the scale region and yield NaNs).
|
||||
@@ -273,13 +277,13 @@ def test_flashmla_fp8_sparse_decode_unified_slot_view():
|
||||
|
||||
# Sparse indices: each batch uses its own disjoint blocks.
|
||||
blocks_per_batch = num_blocks // batch
|
||||
idx = torch.full((batch, 1, topk), -1, device=dev, dtype=torch.int32)
|
||||
idx = torch.full((batch, next_n, topk), -1, device=dev, dtype=torch.int32)
|
||||
for b in range(batch):
|
||||
slots: list[int] = []
|
||||
for blk in range(b * blocks_per_batch, (b + 1) * blocks_per_batch):
|
||||
slots.extend(blk * page + off for off in range(page))
|
||||
slots_t = torch.tensor(slots[:topk], device=dev, dtype=torch.int32)
|
||||
idx[b, 0, : slots_t.numel()] = slots_t
|
||||
idx[b, :, : slots_t.numel()] = slots_t
|
||||
|
||||
def run(kc):
|
||||
meta, num_splits = fm.get_mla_metadata()
|
||||
|
||||
@@ -316,6 +316,48 @@ def test_cp_gather_fp8_shuffled_blocks():
|
||||
assert torch.equal(dst[:, NOPE_DIM:], expected[:, NOPE_DIM:])
|
||||
|
||||
|
||||
def test_cp_gather_fp8_mixes_resident_and_pinned_host_rows():
|
||||
seq_lens = [7, 5]
|
||||
block_size = 4
|
||||
(
|
||||
cache,
|
||||
block_table,
|
||||
workspace_starts,
|
||||
num_reqs,
|
||||
total_tokens,
|
||||
expected,
|
||||
) = _build_test_case(seq_lens, block_size)
|
||||
|
||||
device_cache = cache.clone()
|
||||
host_cache = cache.cpu().pin_memory().view(-1, ENTRY_BYTES)
|
||||
device_rows = torch.arange(host_cache.shape[0], dtype=torch.int32, device="cuda")
|
||||
use_device = device_rows % 2 == 0
|
||||
host_rows = device_rows.clone()
|
||||
device_rows.masked_fill_(~use_device, -1)
|
||||
|
||||
host_cache[use_device.cpu()] = 0
|
||||
device_cache.view(-1, ENTRY_BYTES)[~use_device] = 0
|
||||
dst = torch.empty(
|
||||
total_tokens, NOPE_DIM + ROPE_DIM, dtype=torch.bfloat16, device="cuda"
|
||||
)
|
||||
|
||||
ops.cp_gather_and_upconvert_fp8_kv_cache(
|
||||
device_cache,
|
||||
dst,
|
||||
block_table,
|
||||
workspace_starts,
|
||||
num_reqs,
|
||||
host_cache=host_cache,
|
||||
host_row_ids=host_rows,
|
||||
device_row_ids=device_rows,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(
|
||||
dst[:, :NOPE_DIM], expected[:, :NOPE_DIM], atol=1e-3, rtol=1e-2
|
||||
)
|
||||
assert torch.equal(dst[:, NOPE_DIM:], expected[:, NOPE_DIM:])
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"gather_seq_lens,seq_starts",
|
||||
[
|
||||
|
||||
@@ -234,6 +234,76 @@ def test_fused_norm_rope(num_tokens: int, index_interleave: bool, mla_dtype: str
|
||||
assert (topk == -1).all(), "topk buffer not cleared on indexer layer"
|
||||
|
||||
|
||||
def test_fused_norm_rope_packed_indexer_block_stride():
|
||||
"""Indexer writes use their HMA block stride and independent slot map."""
|
||||
torch.manual_seed(7)
|
||||
dev = "cuda"
|
||||
num_tokens = block_size = 2
|
||||
pos = torch.arange(num_tokens, device=dev, dtype=torch.int64)
|
||||
q_c = torch.randn(num_tokens, Q_LORA, device=dev, dtype=torch.bfloat16)
|
||||
kv_c = torch.randn(num_tokens, KV_LORA, device=dev, dtype=torch.bfloat16)
|
||||
k_pe = torch.randn(num_tokens, ROPE_DIM, device=dev, dtype=torch.bfloat16)
|
||||
qw = torch.randn(Q_LORA, device=dev, dtype=torch.bfloat16)
|
||||
kvw = torch.randn(KV_LORA, device=dev, dtype=torch.bfloat16)
|
||||
ik = torch.randn(num_tokens, INDEX_HEAD_DIM, device=dev, dtype=torch.bfloat16)
|
||||
ikw = torch.randn(INDEX_HEAD_DIM, device=dev, dtype=torch.float32)
|
||||
ikb = torch.randn(INDEX_HEAD_DIM, device=dev, dtype=torch.float32)
|
||||
cos_sin = make_cos_sin(32, ROPE_DIM, dev)
|
||||
|
||||
idx_row = INDEX_HEAD_DIM + INDEX_HEAD_DIM // 128 * 4
|
||||
packed_block_stride = block_size * idx_row + 64
|
||||
backing = torch.zeros(2 * packed_block_stride, device=dev, dtype=torch.uint8)
|
||||
idx_cache = torch.as_strided(
|
||||
backing,
|
||||
(2, block_size, idx_row),
|
||||
(packed_block_stride, idx_row, 1),
|
||||
)
|
||||
mla_cache = torch.zeros(
|
||||
1,
|
||||
block_size,
|
||||
KV_LORA + ROPE_DIM,
|
||||
device=dev,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
topk = torch.zeros(num_tokens, 8, device=dev, dtype=torch.int32)
|
||||
|
||||
K.fused_norm_rope(
|
||||
pos,
|
||||
q_c,
|
||||
qw,
|
||||
EPS,
|
||||
kv_c,
|
||||
kvw,
|
||||
EPS,
|
||||
k_pe,
|
||||
cos_sin,
|
||||
ik,
|
||||
ikw,
|
||||
ikb,
|
||||
EPS,
|
||||
cos_sin,
|
||||
topk,
|
||||
slot_mapping=torch.arange(num_tokens, device=dev, dtype=torch.int64),
|
||||
indexer_slot_mapping=torch.arange(
|
||||
block_size, block_size + num_tokens, device=dev, dtype=torch.int64
|
||||
),
|
||||
indexer_k_cache=idx_cache,
|
||||
mla_kv_cache=mla_cache,
|
||||
has_indexer=True,
|
||||
)
|
||||
|
||||
ik_ref = rope(layer_norm(ik, ikw, ikb), pos, cos_sin, interleave=False)
|
||||
q_ref, s_ref = ue8m0_quant(ik_ref)
|
||||
packed = idx_cache[1].reshape(-1)
|
||||
values = (
|
||||
packed[: block_size * INDEX_HEAD_DIM].view(FP8).view(block_size, INDEX_HEAD_DIM)
|
||||
)
|
||||
scales = packed[block_size * INDEX_HEAD_DIM :].view(torch.float32)
|
||||
assert_fp8(values, q_ref, "packed indexer-K fp8")
|
||||
torch.testing.assert_close(scales, s_ref, rtol=0, atol=0)
|
||||
assert (backing[block_size * idx_row : packed_block_stride] == 0).all()
|
||||
|
||||
|
||||
def test_fused_norm_rope_materializes_qk_without_local_cache_slots():
|
||||
"""DCP non-owner ranks still need valid query shards for query AllGather."""
|
||||
torch.manual_seed(7)
|
||||
|
||||
Reference in New Issue
Block a user