[Bugfix] Add swiglu limits to deepgemm fp8 methods (#41986)
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
+1
@@ -360,6 +360,7 @@ class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod):
|
||||
per_act_token_quant=is_per_token,
|
||||
per_out_ch_quant=is_per_token,
|
||||
block_shape=self.weight_block_size,
|
||||
swiglu_limit=getattr(layer, "swiglu_limit", None),
|
||||
)
|
||||
|
||||
def apply_monolithic(
|
||||
|
||||
+1
@@ -152,6 +152,7 @@ class CompressedTensorsW8A8Mxfp8MoEMethod(CompressedTensorsMoEMethod):
|
||||
a1_scale=layer.w13_input_scale,
|
||||
a2_scale=layer.w2_input_scale,
|
||||
block_shape=self.weight_block_size,
|
||||
swiglu_limit=getattr(layer, "swiglu_limit", None),
|
||||
)
|
||||
|
||||
def maybe_make_prepare_finalize(
|
||||
|
||||
@@ -856,6 +856,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
a1_scale=a1_scale,
|
||||
a2_scale=a2_scale,
|
||||
block_shape=self.weight_block_size,
|
||||
swiglu_limit=getattr(layer, "swiglu_limit", None),
|
||||
)
|
||||
|
||||
# Inject biases into the quant config if the model has them
|
||||
|
||||
@@ -944,6 +944,7 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
|
||||
w2_scale=w2_scale,
|
||||
a1_scale=a1_scale,
|
||||
a2_scale=a2_scale,
|
||||
swiglu_limit=getattr(layer, "swiglu_limit", None),
|
||||
)
|
||||
|
||||
def apply_monolithic(
|
||||
|
||||
@@ -376,6 +376,7 @@ class _Fp8OnlineMoEBase(OnlineMoEMethodBase):
|
||||
a1_scale=a1_scale,
|
||||
a2_scale=a2_scale,
|
||||
block_shape=self.weight_block_size,
|
||||
swiglu_limit=getattr(layer, "swiglu_limit", None),
|
||||
)
|
||||
|
||||
self._maybe_inject_biases(quant_config, layer)
|
||||
|
||||
@@ -221,6 +221,7 @@ class Mxfp8OnlineMoEMethod(OnlineMoEMethodBase):
|
||||
a1_scale=a1_scale,
|
||||
a2_scale=a2_scale,
|
||||
block_shape=self.weight_block_size,
|
||||
swiglu_limit=getattr(layer, "swiglu_limit", None),
|
||||
)
|
||||
|
||||
self._maybe_inject_biases(quant_config, layer)
|
||||
|
||||
@@ -449,6 +449,7 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod):
|
||||
w2_bias=layer.w2_bias,
|
||||
per_act_token_quant=self.input_qscheme == "per_channel",
|
||||
per_out_ch_quant=self.weight_qscheme == "per_channel",
|
||||
gemm1_clamp_limit=getattr(layer, "swiglu_limit", None),
|
||||
)
|
||||
|
||||
def apply(
|
||||
@@ -941,6 +942,7 @@ class QuarkW4A8Fp8MoEMethod(QuarkMoEMethod):
|
||||
w1_scale=layer.w13_weight_scale_2,
|
||||
w2_scale=layer.w2_weight_scale_2,
|
||||
per_out_ch_quant=True,
|
||||
gemm1_clamp_limit=getattr(layer, "swiglu_limit", None),
|
||||
)
|
||||
|
||||
def apply(
|
||||
|
||||
@@ -302,9 +302,11 @@ def _silu_mul_per_token_group_quant_fp8_colmajor(
|
||||
y_s_col_stride: tl.int64,
|
||||
# Information for float8
|
||||
eps,
|
||||
clamp_limit,
|
||||
fp8_min: tl.constexpr,
|
||||
fp8_max: tl.constexpr,
|
||||
use_ue8m0: tl.constexpr,
|
||||
HAS_CLAMP: tl.constexpr,
|
||||
# Meta-parameters
|
||||
GROUP_SIZE: tl.constexpr,
|
||||
BLOCK_M: tl.constexpr,
|
||||
@@ -336,7 +338,16 @@ def _silu_mul_per_token_group_quant_fp8_colmajor(
|
||||
act_in = tl.load(act_in_ptrs)
|
||||
mul_in = tl.load(act_in_ptrs + N_2)
|
||||
|
||||
# silu & mul
|
||||
# silu & mul — match C++ silu_and_mul: clamp in fp32 then store back to the
|
||||
# input dtype, run silu in fp32 then narrow, and do the mul at input
|
||||
# precision so HAS_CLAMP True/False share the same multiplication path.
|
||||
if HAS_CLAMP:
|
||||
act_in = tl.minimum(act_in.to(tl.float32), clamp_limit).to(
|
||||
y_ptr.dtype.element_ty
|
||||
)
|
||||
mul_in = tl.clamp(mul_in.to(tl.float32), -clamp_limit, clamp_limit).to(
|
||||
y_ptr.dtype.element_ty
|
||||
)
|
||||
act_in = act_in.to(tl.float32)
|
||||
one_f32 = tl.cast(1, tl.float32)
|
||||
silu_out = (act_in / (one_f32 + tl.exp(-act_in))).to(y_ptr.dtype.element_ty)
|
||||
@@ -367,6 +378,7 @@ def silu_mul_per_token_group_quant_fp8_colmajor(
|
||||
output: torch.Tensor | None = None, # [M, N // 2]
|
||||
use_ue8m0: bool | None = None,
|
||||
eps: float = 1e-10,
|
||||
clamp_limit: float | None = None,
|
||||
):
|
||||
"""
|
||||
silu+mul + block-fp8 quant with group size 128.
|
||||
@@ -409,6 +421,7 @@ def silu_mul_per_token_group_quant_fp8_colmajor(
|
||||
assert N_2 % BLOCK_N == 0
|
||||
grid = (M // BLOCK_M, N_2 // BLOCK_N)
|
||||
|
||||
has_clamp = clamp_limit is not None
|
||||
_silu_mul_per_token_group_quant_fp8_colmajor[grid](
|
||||
input,
|
||||
output,
|
||||
@@ -417,9 +430,11 @@ def silu_mul_per_token_group_quant_fp8_colmajor(
|
||||
N,
|
||||
output_scales.stride(-1),
|
||||
eps,
|
||||
clamp_limit if has_clamp else 0.0,
|
||||
fp8_min,
|
||||
fp8_max,
|
||||
use_ue8m0,
|
||||
has_clamp,
|
||||
GROUP_SIZE,
|
||||
BLOCK_M,
|
||||
BLOCK_N,
|
||||
|
||||
Reference in New Issue
Block a user