[3/N] HiSparse: host-resident sparse-MLA decode hot-buffering (#53781)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Signed-off-by: Matthew Bonanni <matthew@neuralmagic.com>
Signed-off-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
Matthew Bonanni
2026-09-12 00:36:47 -04:00
committed by GitHub
co-authored by OpenAI Codex Lucas Wilkinson Lucas Wilkinson Nicolò Lucchesi OpenAI Codex Claude Opus 5 Lucas Wilkinson
parent 120ec4ebd2
commit d43bb2f37f
124 changed files with 14262 additions and 906 deletions
+28 -6
View File
@@ -407,6 +407,8 @@ def triton_convert_req_index_to_global_index(
prefill_workspace_request_ids: torch.Tensor | None = None,
prefill_workspace_starts: torch.Tensor | None = None,
return_valid_counts: bool = False,
out: torch.Tensor | None = None,
valid_counts_out: torch.Tensor | None = None,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
"""
out[token_id, indice_id] =
@@ -462,16 +464,36 @@ def triton_convert_req_index_to_global_index(
# contiguous prefix [0, valid_count) and leaves the tail unwritten, so
# pre-fill -1 there. flash_mla_sparse_fwd then bounds attention to
# [:topk_length] == exactly the valid set (no dropped tokens).
if return_valid_counts:
out = torch.full_like(token_indices_c, -1)
if out is None:
out = (
torch.full_like(token_indices_c, -1)
if return_valid_counts
else torch.empty_like(token_indices_c)
)
else:
out = torch.empty_like(token_indices_c)
assert out.dtype == token_indices_c.dtype
assert out.device == token_indices_c.device
assert out.shape == token_indices_c.shape
assert out.is_contiguous()
if return_valid_counts:
out.fill_(-1)
valid_counts: torch.Tensor | None = None
if return_valid_counts:
# Zero-init only matters for the atomic accumulation path.
alloc = torch.empty if single_tile else torch.zeros
valid_counts = alloc(num_tokens, dtype=torch.int32, device=token_indices.device)
if valid_counts_out is None:
# Zero-init only matters for the atomic accumulation path.
alloc = torch.empty if single_tile else torch.zeros
valid_counts = alloc(
num_tokens, dtype=torch.int32, device=token_indices.device
)
else:
assert valid_counts_out.dtype == torch.int32
assert valid_counts_out.device == token_indices.device
assert valid_counts_out.shape == (num_tokens,)
assert valid_counts_out.is_contiguous()
valid_counts = valid_counts_out
if not single_tile:
valid_counts.zero_()
# Prepare prefill pointers
if HAS_PREFILL_WORKSPACE: