[Quantization][Humming] Support MXFP4 weight + block-FP8 activation for MoE (#51332)

Signed-off-by: Elvir Crncevic <elvircrn@gmail.com>
Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
Signed-off-by: Elvir Crnčević <elvircrn@gmail.com>
Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
Co-authored-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Elvir Crnčević
2026-08-25 10:21:00 -04:00
committed by GitHub
co-authored by Claude Opus 4.8 Tyler Michael Smith mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 9c8e90eb26
commit ca29cd48ba
17 changed files with 1594 additions and 171 deletions
@@ -307,7 +307,18 @@ def _humming_input_schema_to_quant_key(
gs = schema.input_scale_group_size
group_shape = GroupShape(row=1, col=gs) if gs > 0 else GroupShape.PER_TOKEN
scale_dtype = MXFP_SCALE_DTYPE if gs > 0 else torch.float32
# Pick the scale dtype the Humming kernel actually consumes. An explicit
# input_scale_dtype always wins. Otherwise infer from the grouping: MX
# microscale activations (group size 32, e.g. MXFP8) carry an e8m0 (uint8)
# scale, while block-FP8 (group size 128) and per-token FP8/int8 carry a
# float32 scale. Getting this right lets a grouped FP8 activation match
# kFp8Dynamic128Sym instead of an unmatchable uint8-scaled key.
if schema.input_scale_dtype is not None:
scale_dtype = _HUMMING_TO_SCALE_DTYPE[schema.input_scale_dtype]
elif gs == 32:
scale_dtype = MXFP_SCALE_DTYPE
else:
scale_dtype = torch.float32
scale = ScaleDesc(dtype=scale_dtype, static=False, group_shape=group_shape)
@@ -553,6 +564,7 @@ def make_humming_moe_quant_config(
quant_dtype: torch.dtype | str | None,
weight_dtype: torch.dtype | str | None,
weight_group_shape: GroupShape | None = None,
activation_group_shape: GroupShape | None = None,
w1_scale: torch.Tensor | None = None,
w2_scale: torch.Tensor | None = None,
w1_zp: torch.Tensor | None = None,
@@ -569,7 +581,14 @@ def make_humming_moe_quant_config(
assert humming_configs is not None
if quant_dtype is None:
a_quant_desc = FusedMoEQuantDesc(dtype=None)
elif activation_group_shape is not None:
# Pre-dispatch quantization.
a_quant_desc = FusedMoEQuantDesc(
dtype=quant_dtype, shape=activation_group_shape
)
else:
# Deferred path: Humming quantizes the activation internally, so the
# descriptor only needs a non-None dtype to mark it as quantized.
shape = GroupShape(row=1, col=-1)
a_quant_desc = FusedMoEQuantDesc(dtype=quant_dtype, shape=shape)
@@ -621,6 +640,19 @@ def get_humming_moe_quant_config(
else:
q_dtype = str(input_schema.a_dtype)
# Block-FP8 (group-128) activations are quantized *before* the EP all-to-all
# dispatch (so FP8 rather than BF16 crosses the interconnect) and consumed by
# Humming as-is.
activation_group_shape: GroupShape | None = None
input_scale_group_size = getattr(input_schema, "input_scale_group_size", 0) or 0
if (
q_dtype is not None
and q_dtype.startswith("float8")
and input_scale_group_size == 128
):
q_dtype = _HUMMING_TO_QUANT_DTYPE.get(input_schema.a_dtype, FP8_DTYPE)
activation_group_shape = GroupShape(row=1, col=input_scale_group_size)
weight_scale_group_size = weight_schema.weight_scale_group_size
weight_scale_group_size_n = weight_schema.weight_scale_group_size_n
weight_group_shape: tuple[int, ...] = ()
@@ -638,6 +670,7 @@ def get_humming_moe_quant_config(
quant_dtype=q_dtype,
weight_dtype=str(weight_schema.b_dtype),
weight_group_shape=weight_group_shape,
activation_group_shape=activation_group_shape,
w1_scale=getattr(layer, "w13_weight_scale", None),
w1_gscale=getattr(layer, "w13_weight_scale_2", None),
w1_zp=getattr(layer, "w13_zero_point", None),