[Kernel]Add norm fused kernel for Step-5 Preview and Muse Spark Open (#62132)

Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
This commit is contained in:
Jee Jee Li
2026-10-17 01:53:00 -07:00
committed by GitHub
parent eb8de8d1a1
commit e56b1603af
10 changed files with 2461 additions and 0 deletions
+110
View File
@@ -339,6 +339,17 @@ def fused_add_rms_norm(
torch.ops._C.fused_add_rms_norm(input, residual, weight, epsilon)
def muse_spark_mul_tanh_soft_cap(
x: torch.Tensor,
multiplier: float,
soft_cap_temp: float,
) -> torch.Tensor:
"""Cast to FP32 and apply ``temp * tanh(x * multiplier / temp)`` in one pass."""
out = torch.empty_like(x, dtype=torch.float32)
torch.ops._C.muse_spark_mul_tanh_soft_cap(out, x, multiplier, soft_cap_temp)
return out
def fused_qk_norm_rope(
qkv: torch.Tensor,
num_heads_q: int,
@@ -2598,6 +2609,105 @@ def reshape_and_cache_flash(
)
def fused_qknorm_rope_cache(
qkv: torch.Tensor,
q_weight: torch.Tensor,
k_weight: torch.Tensor,
cos_sin_cache: torch.Tensor,
positions: torch.Tensor,
slot_mapping: torch.Tensor,
key_cache: torch.Tensor,
value_cache: torch.Tensor,
num_q_heads: int,
num_kv_heads: int,
head_dim: int,
rotary_dim: int,
eps: float,
norm_weight_bias: float = 0.0,
kv_cache_dtype: str = "auto",
k_scale: torch.Tensor | None = None,
v_scale: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""Fuse per-head RMSNorm, partial NeoX RoPE and KV insertion.
Norm weights may be FP32 independently of QKV. A zero rotary dimension
skips RoPE; norm outputs are rounded to QKV dtype before rotation.
"""
if rotary_dim < 0 or rotary_dim > head_dim or rotary_dim % 8:
raise ValueError("rotary_dim must be a multiple of 8 in [0, head_dim]")
q = qkv.new_empty((qkv.shape[0], num_q_heads * head_dim))
k = qkv.new_empty((qkv.shape[0], num_kv_heads * head_dim))
v = torch.empty_like(k)
cos, sin = cos_sin_cache.chunk(2, dim=-1)
torch.ops._C.fused_qknorm_rope_cache(
q,
k,
v,
qkv,
q_weight,
k_weight,
cos,
sin,
positions,
slot_mapping,
key_cache,
value_cache,
head_dim,
num_q_heads,
num_kv_heads,
rotary_dim // 2,
eps,
norm_weight_bias,
kv_cache_dtype,
k_scale,
v_scale,
)
return q, k, v
def fused_muse_spark_qknorm_rope_kv_insert(
qkv: torch.Tensor,
positions: torch.Tensor,
num_heads: int,
num_kv_heads: int,
head_dim: int,
eps: float,
cos_sin_cache: torch.Tensor | None = None,
rotary_dim: int = 0,
slot_mapping: torch.Tensor | None = None,
kv_cache: torch.Tensor | None = None,
block_size: int = 0,
kv_cache_dtype: str = "auto",
k_scale: torch.Tensor | None = None,
v_scale: torch.Tensor | None = None,
q_fp8_out: torch.Tensor | None = None,
q_scale: torch.Tensor | None = None,
) -> None:
"""In-place weightless QK RMSNorm + GPT-J RoPE, optionally writing k/v to cache.
When ``kv_cache`` is given, the caller must skip the attention layer's own KV write.
With ``q_fp8_out``, write scaled E4M3 Q there instead of rewriting Q in ``qkv``.
"""
torch.ops._C.fused_muse_spark_qknorm_rope_kv_insert(
qkv,
positions,
num_heads,
num_kv_heads,
head_dim,
eps,
cos_sin_cache,
rotary_dim,
slot_mapping,
kv_cache,
block_size,
kv_cache_dtype,
k_scale,
v_scale,
q_fp8_out,
q_scale,
)
def fused_minimax_m3_qknorm_rope_kv_insert(
qkv: torch.Tensor,
q_norm_weight: torch.Tensor,