[3/N] HiSparse: host-resident sparse-MLA decode hot-buffering (#53781)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Signed-off-by: Matthew Bonanni <matthew@neuralmagic.com>
Signed-off-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
Matthew Bonanni
2026-09-12 00:36:47 -04:00
committed by GitHub
co-authored by OpenAI Codex Lucas Wilkinson Lucas Wilkinson Nicolò Lucchesi OpenAI Codex Claude Opus 5 Lucas Wilkinson
parent 120ec4ebd2
commit d43bb2f37f
124 changed files with 14262 additions and 906 deletions
@@ -55,6 +55,7 @@ def build_chunked_context(
def test_sparse_context_lengths_do_not_force_gpu_sync(monkeypatch):
"""Sparse MLA must pin computed context lengths before the H2D copy."""
builder = SimpleNamespace(
_context_lens_cpu=torch.empty(2, dtype=torch.int32, pin_memory=True),
chunked_prefill_workspace=torch.empty((2048, 1)),
chunked_prefill_workspace_size=1024,
kv_cache_spec=SimpleNamespace(block_size=BLOCK_SIZE),
@@ -113,6 +114,7 @@ def test_chunks_gather_every_context_row_exactly_once(context_lens, workspace_si
query_lens = [4] * len(context_lens)
metadata = build_chunked_context(context_lens, query_lens, workspace_size)
assert metadata is not None
assert metadata.context_lens.tolist() == context_lens
gathered: dict[int, list[tuple[int, int]]] = {}
previous_request_start = -1
@@ -253,6 +255,7 @@ def test_dcp_chunks_fit_the_per_rank_row_budget():
dcp_local_block_size=interleave,
)
assert metadata is not None
assert metadata.context_lens.tolist() == context_lens
virtual_block_size = interleave * dcp_world_size
local_cursor: dict[int, int] = {}
@@ -330,3 +333,23 @@ def test_dcp_reorg_uses_each_chunks_local_starts():
)
torch.testing.assert_close(reorganized, torch.cat(expected))
def test_chunked_context_metadata_owns_context_lengths():
"""Reusing CPU scratch must not overwrite an earlier batch's context lengths."""
context_lens = torch.tensor([16, 32], dtype=torch.int32)
metadata = build_mla_chunked_context_metadata(
context_lens_cpu=context_lens,
prefill_query_start_loc_cpu=torch.tensor([0, 4, 8], dtype=torch.int32),
chunked_prefill_workspace=torch.empty((64, 1)),
chunked_prefill_workspace_size=64,
block_size=BLOCK_SIZE,
align_chunk_to_block=True,
device=torch.device("cpu"),
dcp_world_size=1,
dcp_local_block_size=1,
dcp_virtual_block_size=1,
)
context_lens.zero_()
assert metadata is not None
assert metadata.context_lens.tolist() == [16, 32]