[Quantization][Humming] Support MXFP4 weight + block-FP8 activation for MoE (#51332)

Signed-off-by: Elvir Crncevic <elvircrn@gmail.com>
Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
Signed-off-by: Elvir Crnčević <elvircrn@gmail.com>
Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
Co-authored-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Elvir Crnčević
2026-08-25 10:21:00 -04:00
committed by GitHub
co-authored by Claude Opus 4.8 Tyler Michael Smith mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 9c8e90eb26
commit ca29cd48ba
17 changed files with 1594 additions and 171 deletions
@@ -201,6 +201,7 @@ def apply_moe_activation(
activation_config: ApplyMoEActivationConfig | None = None,
topk_ids: torch.Tensor | None = None,
expert_map: torch.Tensor | None = None,
valid_rows: torch.Tensor | None = None,
) -> torch.Tensor:
"""Apply MoE activation function.
@@ -256,6 +257,7 @@ def apply_moe_activation(
-1.0
if config.activation_situ_linear_beta is None
else config.activation_situ_linear_beta,
valid_rows,
)
elif activation == MoEActivation.SWIGLUOAI:
torch.ops._C.swigluoai_and_mul(output, input)
@@ -286,3 +288,45 @@ def apply_moe_activation(
raise ValueError(f"Unsupported FusedMoe activation: {activation}")
return output
def situ_and_mul_quant(
output: torch.Tensor,
scale: torch.Tensor,
input: torch.Tensor,
*,
beta: float,
linear_beta: float | None,
group_size: int = 0,
valid_rows: torch.Tensor | None = None,
topk: int = 1,
) -> None:
"""Fused Kimi SITU activation + dynamic FP8 quantization.
Writes the quantized fp8 down-projection input into ``output`` [M, d] and its
float32 scale into ``scale`` (dequant = q * scale), fusing the SITU
activation with the FP8 quant that Humming's w2 GEMM would otherwise perform
in a separate pass -- this avoids materializing (and rounding through) the
bf16 activation_output buffer.
``group_size`` selects the quant granularity: ``0`` gives per-token scales
``scale`` [M, 1]; ``128`` gives k-major block-FP8 group scales ``scale``
[M, d // 128], matching humming ``quant_input(group_size=128, float32)``.
``linear_beta`` <= 0 (or None) means "unset" (up passed through), matching
``SituAndMul(linear_beta=None)``. ``valid_rows`` (int32 scalar tensor) is the
DeepEP v2 contiguous-layout valid *token* count (e.g. ``psum[-1:]``); the
kernel multiplies it by ``topk`` to bound rows, so no host-side cast/multiply
is needed. Padding rows are skipped and receive a benign scale of 1.0. Kept
in this module so that every ``torch.ops._C.situ*`` call lives in one file.
"""
torch.ops._C.situ_and_mul_quant(
output,
scale,
input,
beta,
-1.0 if linear_beta is None else linear_beta,
group_size,
valid_rows,
topk,
)