fused qknorm+rope kernel optimization for SM9.0 (#37376)

Signed-off-by: EricccYang <yangyang4991@gmail.com>
Signed-off-by: Kaicheng Yang <53411596+EricccYang@users.noreply.github.com>
Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Kaicheng Yang
2026-04-12 19:58:37 -07:00
committed by GitHub
co-authored by Claude Sonnet 4.6
parent cae984060f
commit 4beeb0689c
7 changed files with 497 additions and 12 deletions
+2
View File
@@ -435,6 +435,7 @@ def fused_qk_norm_rope(
cos_sin_cache: torch.Tensor,
is_neox: bool,
position_ids: torch.Tensor,
forced_token_heads_per_warp: int = -1,
) -> None:
torch.ops._C.fused_qk_norm_rope(
qkv,
@@ -448,6 +449,7 @@ def fused_qk_norm_rope(
cos_sin_cache,
is_neox,
position_ids,
forced_token_heads_per_warp,
)
@@ -164,6 +164,7 @@ class QkNormRopePattern:
cos_sin_cache=cos_sin_cache,
is_neox=self.is_neox,
position_ids=positions.view(-1),
forced_token_heads_per_warp=-1,
)
result_qkv = result[1]
@@ -168,6 +168,7 @@ class FixFunctionalizationPass(VllmInductorPass):
"cos_sin_cache",
"is_neox",
"position_ids",
"forced_token_heads_per_warp",
)
self.defunctionalize(graph, node, mutated_args=mutated_args, args=args)
elif (