[Performance][DSR1]: Fused RoPE+KVCache+q_concat for MLA (#40392)
Signed-off-by: Rohan138 <rohanpotdar138@gmail.com> Signed-off-by: Rohan Potdar <66227218+Rohan138@users.noreply.github.com> Co-authored-by: ElizaWszola <ewszola@redhat.com>
This commit is contained in:
co-authored by
ElizaWszola
parent
8415bf2cdb
commit
a51376b3f0
@@ -136,8 +136,10 @@ class PassConfig:
|
||||
"""Enable flashinfer allreduce fusion."""
|
||||
fuse_minimax_qk_norm: bool = None # type: ignore[assignment]
|
||||
"""Enable fused allreduce+RMSNorm for MiniMax QK norm."""
|
||||
enable_qk_norm_rope_fusion: bool = False
|
||||
enable_qk_norm_rope_fusion: bool = None # type: ignore[assignment]
|
||||
"""Enable fused Q/K RMSNorm + RoPE pass."""
|
||||
fuse_rope_kvcache_cat_mla: bool = None # type: ignore[assignment]
|
||||
"""Enable fused MLA KV cache update with RoPE."""
|
||||
|
||||
# ROCm/AITER specific fusions
|
||||
fuse_act_padding: bool = None # type: ignore[assignment]
|
||||
@@ -228,6 +230,7 @@ class PassConfig:
|
||||
"fuse_act_padding",
|
||||
"fuse_mla_dual_rms_norm",
|
||||
"fuse_rope_kvcache",
|
||||
"fuse_rope_kvcache_cat_mla",
|
||||
mode="wrap",
|
||||
)
|
||||
@classmethod
|
||||
@@ -285,6 +288,12 @@ class PassConfig:
|
||||
"The fusion will be disabled."
|
||||
)
|
||||
self.fuse_rope_kvcache = False
|
||||
if self.fuse_rope_kvcache_cat_mla and not current_platform.is_cuda_alike():
|
||||
logger.warning_once(
|
||||
"MLA KV cache update with RoPE fusion enabled but the "
|
||||
"current platform is not CUDA or ROCm. The fusion will be disabled."
|
||||
)
|
||||
self.fuse_rope_kvcache_cat_mla = False
|
||||
|
||||
def log_enabled_passes(self) -> None:
|
||||
"""
|
||||
|
||||
Reference in New Issue
Block a user