[Kimi Perf] Group fp8 mla cahche insertion, 4~6x kernel level performance improvement for small batch (#55356)

Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
Wentao Ye
2026-09-11 08:16:57 -07:00
committed by GitHub
parent 22258a26bc
commit 1e1060f998
6 changed files with 216 additions and 39 deletions
+5 -3
View File
@@ -976,14 +976,16 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C_cache_ops, ops) {
" str kv_cache_dtype,"
" Tensor scale) -> ()");
// Grouped concat_and_cache_mla across all layers (bf16 only). Each
// layer's cache base pointer is read from kv_cache_ptrs.
// Grouped concat_and_cache_mla across all layers. Each layer's cache base
// pointer and optional plain-FP8 scale are read from device tensors.
ops.def(
"concat_and_cache_mla_grouped(Tensor kv_c, Tensor k_pe,"
" Tensor kv_cache_ptrs,"
" Tensor slot_mapping,"
" int block_size, int block_stride,"
" int entry_stride) -> ()");
" int entry_stride,"
" Tensor? kv_scales=None,"
" str kv_cache_dtype='auto') -> ()");
// Rotate Q and K, then write to kv cache for MLA
ops.def(