[Kernel] Optimize long-context MLA cache gathers (#51739)

Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Yongye Zhu
2026-08-10 22:23:43 -07:00
committed by GitHub
co-authored by OpenAI Codex Claude Opus 5
parent 65b7662d3f
commit c76a425278
6 changed files with 808 additions and 233 deletions
@@ -2574,7 +2574,7 @@ class MLACommonBaseImpl(MLAAttentionImpl[A], Generic[A]):
# FP8 path: gather cache without dequantization
ops.cp_gather_cache(
src_cache=kv_c_and_k_pe_cache,
dst=workspace,
dst=workspace[:toks],
block_table=block_table,
cu_seq_lens=chunk.cu_seq_lens,
batch_size=chunk.num_requests,
@@ -2683,7 +2683,7 @@ class MLACommonBaseImpl(MLAAttentionImpl[A], Generic[A]):
else:
ops.cp_gather_cache(
src_cache=kv_c_and_k_pe_cache,
dst=workspace,
dst=workspace[:toks],
block_table=block_table,
cu_seq_lens=padded_local_cu_seq_lens,
batch_size=chunk.num_requests,