[Kernel]Add norm fused kernel for Step-5 Preview and Muse Spark Open (#62132)
Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
This commit is contained in:
@@ -339,6 +339,17 @@ def fused_add_rms_norm(
|
||||
torch.ops._C.fused_add_rms_norm(input, residual, weight, epsilon)
|
||||
|
||||
|
||||
def muse_spark_mul_tanh_soft_cap(
|
||||
x: torch.Tensor,
|
||||
multiplier: float,
|
||||
soft_cap_temp: float,
|
||||
) -> torch.Tensor:
|
||||
"""Cast to FP32 and apply ``temp * tanh(x * multiplier / temp)`` in one pass."""
|
||||
out = torch.empty_like(x, dtype=torch.float32)
|
||||
torch.ops._C.muse_spark_mul_tanh_soft_cap(out, x, multiplier, soft_cap_temp)
|
||||
return out
|
||||
|
||||
|
||||
def fused_qk_norm_rope(
|
||||
qkv: torch.Tensor,
|
||||
num_heads_q: int,
|
||||
@@ -2598,6 +2609,105 @@ def reshape_and_cache_flash(
|
||||
)
|
||||
|
||||
|
||||
def fused_qknorm_rope_cache(
|
||||
qkv: torch.Tensor,
|
||||
q_weight: torch.Tensor,
|
||||
k_weight: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
slot_mapping: torch.Tensor,
|
||||
key_cache: torch.Tensor,
|
||||
value_cache: torch.Tensor,
|
||||
num_q_heads: int,
|
||||
num_kv_heads: int,
|
||||
head_dim: int,
|
||||
rotary_dim: int,
|
||||
eps: float,
|
||||
norm_weight_bias: float = 0.0,
|
||||
kv_cache_dtype: str = "auto",
|
||||
k_scale: torch.Tensor | None = None,
|
||||
v_scale: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Fuse per-head RMSNorm, partial NeoX RoPE and KV insertion.
|
||||
|
||||
Norm weights may be FP32 independently of QKV. A zero rotary dimension
|
||||
skips RoPE; norm outputs are rounded to QKV dtype before rotation.
|
||||
"""
|
||||
if rotary_dim < 0 or rotary_dim > head_dim or rotary_dim % 8:
|
||||
raise ValueError("rotary_dim must be a multiple of 8 in [0, head_dim]")
|
||||
q = qkv.new_empty((qkv.shape[0], num_q_heads * head_dim))
|
||||
k = qkv.new_empty((qkv.shape[0], num_kv_heads * head_dim))
|
||||
v = torch.empty_like(k)
|
||||
cos, sin = cos_sin_cache.chunk(2, dim=-1)
|
||||
torch.ops._C.fused_qknorm_rope_cache(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
qkv,
|
||||
q_weight,
|
||||
k_weight,
|
||||
cos,
|
||||
sin,
|
||||
positions,
|
||||
slot_mapping,
|
||||
key_cache,
|
||||
value_cache,
|
||||
head_dim,
|
||||
num_q_heads,
|
||||
num_kv_heads,
|
||||
rotary_dim // 2,
|
||||
eps,
|
||||
norm_weight_bias,
|
||||
kv_cache_dtype,
|
||||
k_scale,
|
||||
v_scale,
|
||||
)
|
||||
return q, k, v
|
||||
|
||||
|
||||
def fused_muse_spark_qknorm_rope_kv_insert(
|
||||
qkv: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
num_heads: int,
|
||||
num_kv_heads: int,
|
||||
head_dim: int,
|
||||
eps: float,
|
||||
cos_sin_cache: torch.Tensor | None = None,
|
||||
rotary_dim: int = 0,
|
||||
slot_mapping: torch.Tensor | None = None,
|
||||
kv_cache: torch.Tensor | None = None,
|
||||
block_size: int = 0,
|
||||
kv_cache_dtype: str = "auto",
|
||||
k_scale: torch.Tensor | None = None,
|
||||
v_scale: torch.Tensor | None = None,
|
||||
q_fp8_out: torch.Tensor | None = None,
|
||||
q_scale: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
"""In-place weightless QK RMSNorm + GPT-J RoPE, optionally writing k/v to cache.
|
||||
|
||||
When ``kv_cache`` is given, the caller must skip the attention layer's own KV write.
|
||||
With ``q_fp8_out``, write scaled E4M3 Q there instead of rewriting Q in ``qkv``.
|
||||
"""
|
||||
torch.ops._C.fused_muse_spark_qknorm_rope_kv_insert(
|
||||
qkv,
|
||||
positions,
|
||||
num_heads,
|
||||
num_kv_heads,
|
||||
head_dim,
|
||||
eps,
|
||||
cos_sin_cache,
|
||||
rotary_dim,
|
||||
slot_mapping,
|
||||
kv_cache,
|
||||
block_size,
|
||||
kv_cache_dtype,
|
||||
k_scale,
|
||||
v_scale,
|
||||
q_fp8_out,
|
||||
q_scale,
|
||||
)
|
||||
|
||||
|
||||
def fused_minimax_m3_qknorm_rope_kv_insert(
|
||||
qkv: torch.Tensor,
|
||||
q_norm_weight: torch.Tensor,
|
||||
|
||||
Reference in New Issue
Block a user