[3/N] HiSparse: host-resident sparse-MLA decode hot-buffering (#53781)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com> Signed-off-by: Matthew Bonanni <matthew@neuralmagic.com> Signed-off-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Lucas Wilkinson <lwilkinson@neuralmagic.com> Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
Lucas Wilkinson
Lucas Wilkinson
Nicolò Lucchesi
OpenAI Codex
Claude Opus 5
Lucas Wilkinson
parent
120ec4ebd2
commit
d43bb2f37f
+15
-1
@@ -3008,6 +3008,9 @@ def cp_gather_and_upconvert_fp8_kv_cache(
|
||||
workspace_starts: torch.Tensor,
|
||||
batch_size: int,
|
||||
seq_starts: torch.Tensor | None = None,
|
||||
host_cache: torch.Tensor | None = None,
|
||||
host_row_ids: torch.Tensor | None = None,
|
||||
device_row_ids: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
"""Gather and upconvert FP8 KV cache to BF16 workspace.
|
||||
|
||||
@@ -3018,9 +3021,20 @@ def cp_gather_and_upconvert_fp8_kv_cache(
|
||||
workspace_starts: Workspace start offsets [num_reqs]
|
||||
batch_size: Number of requests
|
||||
seq_starts: Optional source sequence offsets [num_reqs]
|
||||
host_cache: Optional pinned host rows used for non-resident entries
|
||||
host_row_ids: Host source row for each remapped cache row
|
||||
device_row_ids: Device source row, or -1, for each remapped cache row
|
||||
"""
|
||||
torch.ops._C_cache_ops.cp_gather_and_upconvert_fp8_kv_cache(
|
||||
src_cache, dst, block_table, workspace_starts, batch_size, seq_starts
|
||||
src_cache,
|
||||
dst,
|
||||
block_table,
|
||||
workspace_starts,
|
||||
batch_size,
|
||||
seq_starts,
|
||||
host_cache,
|
||||
host_row_ids,
|
||||
device_row_ids,
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user