[Model] Add MiniMax M3 support (#45381)
Signed-off-by: youkaichao <youkaichao@gmail.com> Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Jee Jee Li <pandaleefree@gmail.com> Signed-off-by: functionstackx <47992694+functionstackx@users.noreply.github.com> Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: functionstackx <47992694+functionstackx@users.noreply.github.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
This commit is contained in:
co-authored by
OpenAI Codex
Isotr0py
Thien Tran
Bugen Zhao
Jee Jee Li
Roger Wang
functionstackx
Yongye Zhu
Jee Jee Li
parent
a3195fab7b
commit
0a1c5034f5
@@ -2614,6 +2614,70 @@ def reshape_and_cache_flash(
|
||||
)
|
||||
|
||||
|
||||
def fused_minimax_m3_qknorm_rope_kv_insert(
|
||||
qkv: torch.Tensor,
|
||||
q_norm_weight: torch.Tensor,
|
||||
k_norm_weight: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
num_heads: int,
|
||||
num_kv_heads: int,
|
||||
rotary_dim: int,
|
||||
eps: float,
|
||||
index_q_norm_weight: torch.Tensor | None = None,
|
||||
index_k_norm_weight: torch.Tensor | None = None,
|
||||
num_index_heads: int = 0,
|
||||
slot_mapping: torch.Tensor | None = None,
|
||||
index_slot_mapping: torch.Tensor | None = None,
|
||||
kv_cache: torch.Tensor | None = None,
|
||||
index_cache: torch.Tensor | None = None,
|
||||
block_size: int = 0,
|
||||
q_out: torch.Tensor | None = None,
|
||||
index_q_out: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
"""Fused MiniMax-M3 attention pre-processing (in-place).
|
||||
|
||||
Applies Gemma RMSNorm + partial NeoX RoPE to ``qkv`` in place. ``qkv`` is a
|
||||
single fused tensor:
|
||||
|
||||
- dense layer (``num_index_heads == 0``): ``[q | k | v]``;
|
||||
- sparse layer (``num_index_heads > 0``): ``[q | k | v | index_q |
|
||||
index_k]`` — the index branch is read straight out of ``qkv``.
|
||||
|
||||
When ``kv_cache`` is given (sparse serving), also scatter-inserts the
|
||||
normed/roped k & v into the paged bf16 KV cache by ``slot_mapping`` and the
|
||||
index key into ``index_cache`` by ``index_slot_mapping``. If
|
||||
``index_slot_mapping`` is omitted, ``slot_mapping`` is used for both caches.
|
||||
|
||||
If ``q_out`` / ``index_q_out`` (contiguous ``[N, nq*128]`` / ``[N,
|
||||
niq*128]``) are given, the normed/roped q / index_q are written there
|
||||
instead of in place — folding the de-interleave into this kernel's store so
|
||||
callers skip a separate ``.contiguous()`` copy before the SM100 sparse
|
||||
attention's flat TMA descriptor.
|
||||
"""
|
||||
torch.ops._C.fused_minimax_m3_qknorm_rope_kv_insert(
|
||||
qkv,
|
||||
q_norm_weight,
|
||||
k_norm_weight,
|
||||
cos_sin_cache,
|
||||
positions,
|
||||
num_heads,
|
||||
num_kv_heads,
|
||||
rotary_dim,
|
||||
eps,
|
||||
index_q_norm_weight,
|
||||
index_k_norm_weight,
|
||||
num_index_heads,
|
||||
slot_mapping,
|
||||
index_slot_mapping,
|
||||
kv_cache,
|
||||
index_cache,
|
||||
block_size,
|
||||
q_out,
|
||||
index_q_out,
|
||||
)
|
||||
|
||||
|
||||
def concat_and_cache_mla(
|
||||
kv_c: torch.Tensor,
|
||||
k_pe: torch.Tensor,
|
||||
|
||||
Reference in New Issue
Block a user