[Quantization][Humming] Support MXFP4 weight + block-FP8 activation for MoE (#51332)

Signed-off-by: Elvir Crncevic <elvircrn@gmail.com>
Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
Signed-off-by: Elvir Crnčević <elvircrn@gmail.com>
Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
Co-authored-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Elvir Crnčević
2026-08-25 10:21:00 -04:00
committed by GitHub
co-authored by Claude Opus 4.8 Tyler Michael Smith mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 9c8e90eb26
commit ca29cd48ba
17 changed files with 1594 additions and 171 deletions
@@ -201,6 +201,7 @@ def apply_moe_activation(
activation_config: ApplyMoEActivationConfig | None = None,
topk_ids: torch.Tensor | None = None,
expert_map: torch.Tensor | None = None,
valid_rows: torch.Tensor | None = None,
) -> torch.Tensor:
"""Apply MoE activation function.
@@ -256,6 +257,7 @@ def apply_moe_activation(
-1.0
if config.activation_situ_linear_beta is None
else config.activation_situ_linear_beta,
valid_rows,
)
elif activation == MoEActivation.SWIGLUOAI:
torch.ops._C.swigluoai_and_mul(output, input)
@@ -286,3 +288,45 @@ def apply_moe_activation(
raise ValueError(f"Unsupported FusedMoe activation: {activation}")
return output
def situ_and_mul_quant(
output: torch.Tensor,
scale: torch.Tensor,
input: torch.Tensor,
*,
beta: float,
linear_beta: float | None,
group_size: int = 0,
valid_rows: torch.Tensor | None = None,
topk: int = 1,
) -> None:
"""Fused Kimi SITU activation + dynamic FP8 quantization.
Writes the quantized fp8 down-projection input into ``output`` [M, d] and its
float32 scale into ``scale`` (dequant = q * scale), fusing the SITU
activation with the FP8 quant that Humming's w2 GEMM would otherwise perform
in a separate pass -- this avoids materializing (and rounding through) the
bf16 activation_output buffer.
``group_size`` selects the quant granularity: ``0`` gives per-token scales
``scale`` [M, 1]; ``128`` gives k-major block-FP8 group scales ``scale``
[M, d // 128], matching humming ``quant_input(group_size=128, float32)``.
``linear_beta`` <= 0 (or None) means "unset" (up passed through), matching
``SituAndMul(linear_beta=None)``. ``valid_rows`` (int32 scalar tensor) is the
DeepEP v2 contiguous-layout valid *token* count (e.g. ``psum[-1:]``); the
kernel multiplies it by ``topk`` to bound rows, so no host-side cast/multiply
is needed. Padding rows are skipped and receive a benign scale of 1.0. Kept
in this module so that every ``torch.ops._C.situ*`` call lives in one file.
"""
torch.ops._C.situ_and_mul_quant(
output,
scale,
input,
beta,
-1.0 if linear_beta is None else linear_beta,
group_size,
valid_rows,
topk,
)
@@ -525,7 +525,9 @@ def deepgemm_moe_permute(
expert_num_tokens = None
if expert_tokens_meta is not None:
expert_num_tokens = expert_tokens_meta.expert_num_tokens
else:
if expert_num_tokens is None:
# A cudagraph-decode carrier may hold only psum_recv_per_rank and leave
# expert_num_tokens None; count locally as in the meta-absent case.
expert_num_tokens = count_expert_num_tokens(
topk_ids, local_num_experts, expert_map
)
@@ -668,6 +668,7 @@ class NaiveBatchedExperts(mk.FusedMoEExpertsModular):
assert hidden_states.dim() == 3
assert expert_tokens_meta is not None
expert_num_tokens = expert_tokens_meta.expert_num_tokens
assert expert_num_tokens is not None
num_local_experts = w1.size(0)
assert num_local_experts == w1.size(0), f"{num_local_experts} == {w1.size(0)}"
@@ -119,6 +119,37 @@ def get_humming_moe_gemm_type() -> str:
return gemm_type
def _fixup_moe_tuning_config(tuning_config: list, max_k_block: int = 128) -> None:
"""Fix up each MoE tile in place: cap the K-block and widen warp-N.
- block_shape[2] (K-block) > ``max_k_block``: the driver rejects the TMA
descriptor at launch (CUDA_ERROR_MISALIGNED_ADDRESS). Cap at 128, which
Humming already uses for larger M.
- warp_shape[1] (warp-N) < 32: block-FP8 (group-128) activations route the
w13 (gate/up) GEMM to a tuning table that pins warp-N to 16, under-filling
the Hopper WGMMA N dimension and corrupting the GEMM output (gsm8k
0.94 -> 0.89). Widen to 32 whenever block_n % 32 == 0 -- on every tile,
not just the K-capped ones. w2 (down) already uses warp_n=32.
"""
logger.info_once("Attempting to override humming GEMM config")
for entry in tuning_config:
config = entry[2]
block_shape = config.get("block_shape")
if not (block_shape and len(block_shape) == 3):
continue
block_m, block_n, block_k = block_shape
logger.info_once(f"Overriding humming GEMM config. Previous config\n: {config}")
if block_k > max_k_block:
config["block_shape"] = [block_m, block_n, max_k_block]
warp_shape = config.get("warp_shape")
if warp_shape and warp_shape[1] < 32 and block_n % 32 == 0:
config["warp_shape"] = [warp_shape[0], 32, warp_shape[2]]
logger.info_once(f"Overridden humming GEMM config. Current config\n: {config}")
class HummingExpertsBase(mk.FusedMoEExpertsModular):
def __init__(
self,
@@ -170,6 +201,10 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
gemm_type=self.humming_gemm_type(),
)
self.compute_config_str = json.dumps(self.compute_config)
# Fix up the heuristic-chosen tiles before freezing to JSON -- see
# _fixup_moe_tuning_config.
_fixup_moe_tuning_config(self.w13_tuning_config)
_fixup_moe_tuning_config(self.w2_tuning_config)
self.w13_tuning_config_str = json.dumps(self.w13_tuning_config)
self.w2_tuning_config_str = json.dumps(self.w2_tuning_config)
@@ -178,12 +213,17 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
sublayer_name: str,
inputs: torch.Tensor,
quanted_input: torch.Tensor | None,
input_scale: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor | None]:
from vllm.utils.humming import may_quant_input
# input_scale is set for block-FP8 (group-128) activations that were
# quantized before the EP dispatch: may_quant_input then skips the
# redundant w13 quantization and forwards the pre-computed scale.
return may_quant_input(
self.humming_configs[sublayer_name],
inputs=inputs,
input_scale=input_scale,
quanted_input=quanted_input,
)
@@ -263,6 +303,10 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
(kMxfp4Static, kMxfp4Dynamic),
(kMxfp4Static, kMxfp8Dynamic),
(kMxfp4Static, kFp8DynamicTokenSym),
# MXFP4 weight (group-32 e8m0) with block-FP8 activation
# (group-128 float32). Runs via WGMMA software dequant, so it
# works on Hopper (SM90/H200) as well as Blackwell.
(kMxfp4Static, kFp8Dynamic128Sym),
(kNvfp4Static, None),
(kNvfp4Static, kFp8DynamicTokenSym),
(kMxfp8Static, None),
@@ -299,21 +343,43 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
and _is_supported_wna16_weight_key(weight_key)
)
def _prequantizes_dispatch_activation(self) -> bool:
"""
Whether the prepare/finalize step should quantize activations before
the (EP all-to-all) dispatch instead of leaving it to Humming.
This is enabled only for block-FP8 (group-128) activations: quantizing
to FP8 before dispatch sends FP8 rather than BF16 over the interconnect,
and Humming then consumes the pre-quantized FP8 + scale directly (see
the apply() methods, which forward the dispatch scale into
HummingMethod.may_quant_input, and may_quant_input itself, which is a
no-op when an input scale is already supplied). The scale layout
produced by vLLM's block-FP8 quantization ([M, K // 128] float32,
row-major) matches what the Humming WGMMA grouped GEMM expects.
"""
quant_config = self.quant_config
return (
quant_config.is_block_quantized
and quant_config.quant_dtype == current_platform.fp8_dtype()
)
@property
def expects_unquantized_inputs(self) -> bool:
"""
Humming kernels handle input quantization internally in apply().
Whether the prepare/finalize step should defer input quantization to
the experts (by setting defer_input_quant=True and passing unquantized
inputs).
This property tells the prepare/finalize step to skip input
quantization (by setting defer_input_quant=True) and pass
unquantized inputs to the experts. This prevents double
quantization: once in prepare and once in Humming's apply().
Humming normally quantizes inputs internally via
HummingMethod.may_quant_input() in apply(), so we defer quantization
(return True) to avoid quantizing twice -- once in prepare and once in
Humming's apply().
Returns:
True to indicate that this expert expects unquantized inputs
and will handle quantization internally.
The exception is block-FP8 (group-128) activations, which are quantized
before the dispatch to save interconnect bandwidth (see
_prequantizes_dispatch_activation): for those we must NOT defer.
"""
return True
return not self._prequantizes_dispatch_activation()
@staticmethod
def _supports_current_device() -> bool:
@@ -585,6 +651,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
activation: MoEActivation,
output: torch.Tensor,
input: torch.Tensor,
valid_rows: torch.Tensor | None = None,
) -> None:
activation_config = self.activation_config
if (
@@ -601,8 +668,84 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
activation=activation,
input=input,
output=output,
valid_rows=valid_rows,
)
def fused_situ_quant_enabled(self, activation: MoEActivation) -> bool:
"""Whether the SITU activation + w2 quant can be fused into one kernel.
Fused only for k-major block-FP8 group-128 e4m3 with float32 scales --
the sole layout situ_and_mul_quant supports. A float32 as_dtype rules out
MXMMA (which uses e8m0 m-major scales), so a group-128 scale is
guaranteed k-major. Everything else (m-major MXFP8, other group sizes,
16-bit passthrough, non-SITU) falls back to the separate
situ_and_mul + quantize_input path.
"""
if activation != MoEActivation.SITU:
return False
w2cfg = self.humming_configs["w2"]
# Report the specific blocker once so a silent fallback to the unfused
# situ_and_mul + quant pair is never a mystery in a trace.
reason: str | None = None
if not (w2cfg.a_dtype.num_bits == 8 and str(w2cfg.a_dtype) == "float8e4m3"):
reason = f"w2 a_dtype is {w2cfg.a_dtype} (need float8e4m3)"
elif w2cfg.input_scale_group_size != 128:
reason = (
f"w2 input_scale_group_size is {w2cfg.input_scale_group_size} "
"(need 128)"
)
elif not (w2cfg.as_dtype is not None and str(w2cfg.as_dtype) == "float32"):
reason = f"w2 as_dtype is {w2cfg.as_dtype} (need float32, k-major)"
if reason is not None:
logger.warning_once(
"Humming fused SITU+FP8 quant disabled, using unfused "
"situ_and_mul + quant: %s",
reason,
)
return False
return True
def fused_situ_quant(
self,
gate_up_output: torch.Tensor,
quanted_down_input: torch.Tensor,
valid_rows: torch.Tensor | None,
topk: int = 1,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Run the fused SITU activation + FP8 quant for the w2 input.
Returns ``(quanted_down_input, input_scale)`` in the same layout the
unfused ``apply_activation`` + ``quantize_input("w2")`` pair produces: an
fp8 [M, d] tensor plus a k-major block-FP8 float32 [M, d // 128] group
scale. Only reached when ``fused_situ_quant_enabled`` (group_size 128).
"""
from vllm.model_executor.layers.fused_moe.activation import (
situ_and_mul_quant,
)
cfg = self.activation_config
assert cfg.activation_situ_beta is not None, (
"SITU requires activation_situ_beta from FusedMoEConfig"
)
group_size = self.humming_configs["w2"].input_scale_group_size
m, d = quanted_down_input.size(0), quanted_down_input.size(1)
input_scale = torch.empty(
(m, d // group_size),
dtype=torch.float32,
device=quanted_down_input.device,
)
situ_and_mul_quant(
quanted_down_input,
input_scale,
gate_up_output,
beta=cfg.activation_situ_beta,
linear_beta=cfg.activation_situ_linear_beta,
group_size=group_size,
valid_rows=valid_rows,
topk=topk,
)
return quanted_down_input, input_scale
class HummingIndexedExperts(HummingExpertsBase):
def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
@@ -684,10 +827,12 @@ class HummingIndexedExperts(HummingExpertsBase):
"""
Standard apply implementation for Humming indexed experts.
Humming performs activation quantization internally and consumes the
weights supplied by the modular kernel interface.
Results are written to the output buffer supplied by the modular
kernel, which may alias workspace13.
Note: Humming kernels handle weights internally through the layer
object, so w1, w2, a2_scale are unused. a1q_scale is None on the usual
path (Humming quantizes the w13 input itself); for block-FP8 activations
it carries the scale computed before dispatch, which is forwarded to
may_quant_input so Humming skips the redundant w13 quantization. The
output is written into workspace13 via the buffer management.
"""
assert not apply_router_weight_on_input
@@ -710,6 +855,7 @@ class HummingIndexedExperts(HummingExpertsBase):
inputs, input_scale = self.quantize_input(
"w13",
inputs=hidden_states,
input_scale=a1q_scale,
quanted_input=buffers.get("quanted_gate_up_input", None),
)
@@ -722,17 +868,45 @@ class HummingIndexedExperts(HummingExpertsBase):
**moe_kwargs1,
)
self.apply_activation(
activation=activation,
input=buffers["gate_up_output"],
output=buffers["activation_output"],
)
# psum[-1:] is the DeepEP valid *token* count as a zero-cost int32 view.
# situ (rows = tokens*topk) multiplies by topk on-device; mul_sum bounds
# on tokens, so both read this pointer directly -- no host cast/multiply.
valid_tokens = None
topk = topk_ids.size(1)
if (
expert_tokens_meta is not None
and expert_tokens_meta.psum_recv_per_rank is not None
):
valid_tokens = expert_tokens_meta.psum_recv_per_rank[-1:]
inputs, input_scale = self.quantize_input(
"w2",
inputs=buffers["activation_output"],
quanted_input=buffers.get("quanted_down_input", None),
)
if self.fused_situ_quant_enabled(activation):
# Fused SITU + FP8 quant (per-token or block-FP8 group-128) straight
# into the w2 input, skipping the bf16 activation_output round-trip.
inputs, input_scale = self.fused_situ_quant(
gate_up_output=buffers["gate_up_output"],
quanted_down_input=buffers["quanted_down_input"],
valid_rows=valid_tokens,
topk=topk,
)
else:
# Fallback situ_and_mul takes int64 row counts (tokens*topk).
valid_rows = (
valid_tokens.to(torch.int64) * topk
if valid_tokens is not None
else None
)
self.apply_activation(
activation=activation,
input=buffers["gate_up_output"],
output=buffers["activation_output"],
valid_rows=valid_rows,
)
inputs, input_scale = self.quantize_input(
"w2",
inputs=buffers["activation_output"],
quanted_input=buffers.get("quanted_down_input", None),
)
self.humming_forward(
"w2",
@@ -743,12 +917,16 @@ class HummingIndexedExperts(HummingExpertsBase):
**moe_kwargs2,
)
# expert_map masks any non-local id; num_valid_tokens bounds the
# persistent kernel to the real token rows [0, num_recv) so the padding
# tail is never iterated (CUDA-graph-safe device scalar).
moe_fused_mul_sum(
inputs=buffers["down_output"].view(*topk_ids.shape, -1),
topk_weights=topk_weights,
topk_ids=topk_ids,
expert_map=expert_map,
outputs=output,
num_valid_tokens=valid_tokens,
)
@@ -787,10 +965,13 @@ class HummingGroupedExperts(HummingExpertsBase):
"""
Standard apply implementation for Humming grouped experts.
Humming performs activation quantization internally and consumes the
weights supplied by the modular kernel interface.
Results are written to the output buffer supplied by the modular
kernel, which may alias workspace13.
Note: Humming kernels handle weights internally through the layer
object, so w1, w2, a2_scale are unused. a1q_scale is None on the usual
path (Humming quantizes the w13 input itself); for block-FP8 activations
it carries the scale computed before dispatch. It is permuted alongside
the tokens by moe_permute and forwarded to may_quant_input so Humming
skips the redundant w13 quantization. The output is written into
workspace13 via the buffer management.
"""
assert not apply_router_weight_on_input
@@ -805,9 +986,9 @@ class HummingGroupedExperts(HummingExpertsBase):
)
buffers["output"] = output
hidden_states, _, expert_first_token_offset, inv_perm, _ = moe_permute(
hidden_states, a1q_scale, expert_first_token_offset, inv_perm, _ = moe_permute(
hidden_states=hidden_states,
a1q_scale=None,
a1q_scale=a1q_scale,
topk_ids=topk_ids,
n_expert=global_num_experts,
n_local_expert=self.num_experts,
@@ -818,6 +999,7 @@ class HummingGroupedExperts(HummingExpertsBase):
inputs, input_scale = self.quantize_input(
"w13",
inputs=hidden_states,
input_scale=a1q_scale,
quanted_input=buffers.get("quanted_gate_up_input", None),
)
@@ -901,15 +1083,21 @@ class BatchedHummingGroupedExperts(HummingExpertsBase):
"""
Standard apply implementation for Humming batched grouped experts.
Humming performs activation quantization internally and consumes the
weights supplied by the modular kernel interface.
Results are written to the output buffer supplied by the modular
kernel, which may alias workspace13.
Note: Humming kernels handle weights internally through the layer
object, so w1, w2, a2_scale are unused. a1q_scale is None on the usual
path (Humming quantizes the w13 input itself); for block-FP8 activations
it carries the scale computed before dispatch, which is forwarded to
may_quant_input so Humming skips the redundant w13 quantization. The
output is written into workspace13 via the buffer management.
"""
assert not apply_router_weight_on_input
assert expert_tokens_meta is not None
hidden_states = hidden_states.view(-1, hidden_states.size(-1))
# Keep the (batched) block-FP8 scale row-aligned with the flattened
# [num_experts * max_tokens, K] hidden states above.
if a1q_scale is not None and a1q_scale.dim() == 3:
a1q_scale = a1q_scale.view(-1, a1q_scale.size(-1))
valid_shape_m = self.estimate_local_valid_shape_m(topk_ids)
expert_num_tokens = expert_tokens_meta.expert_num_tokens
@@ -925,6 +1113,7 @@ class BatchedHummingGroupedExperts(HummingExpertsBase):
inputs, input_scale = self.quantize_input(
"w13",
inputs=hidden_states,
input_scale=a1q_scale,
quanted_input=buffers.get("quanted_gate_up_input", None),
)
@@ -100,8 +100,9 @@ class ExpertTokensMetadata:
Metadata regarding expert-token routing.
"""
expert_num_tokens: torch.Tensor
expert_num_tokens: torch.Tensor | None
expert_num_tokens_cpu: torch.Tensor | None
psum_recv_per_rank: torch.Tensor | None = None
@staticmethod
def make_from_list(
@@ -901,6 +902,7 @@ class FusedMoEExpertsModular(FusedMoEExperts):
*,
topk_ids: torch.Tensor | None = None,
expert_map: torch.Tensor | None = None,
valid_rows: torch.Tensor | None = None,
) -> None:
apply_moe_activation(
activation,
@@ -909,6 +911,7 @@ class FusedMoEExpertsModular(FusedMoEExperts):
activation_config=self.activation_config,
topk_ids=topk_ids,
expert_map=expert_map,
valid_rows=valid_rows,
)
@abstractmethod
@@ -3,7 +3,6 @@
import torch
from torch._subclasses.fake_tensor import FakeTensor
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
@@ -14,122 +13,75 @@ def moe_fused_mul_sum_kernel(
outputs_ptr,
top_ids_ptr,
expert_map_ptr,
num_tokens,
num_valid_tokens_ptr,
stride_m,
has_topk_ids: tl.constexpr,
has_expert_map: tl.constexpr,
has_num_valid: tl.constexpr,
top_k: tl.constexpr,
size: tl.constexpr,
BLOCK_M: tl.constexpr,
hidden_size: tl.constexpr,
BLOCK_K: tl.constexpr,
):
pid_k = tl.program_id(0)
pid_m = tl.program_id(1)
# One CTA owns one token's output row. Resolve which slots to sum once per
# row (the decision is invariant over hidden tiles); the `if take` branch is
# block-uniform, so padding slots are skipped rather than masked-and-loaded.
pid_m = tl.program_id(0)
# Bound to the real recv rows [0, num_recv). Under cudagraph decode the grid
# is the static padded token count, and padding rows past num_recv carry
# stale ids and stale NaN in `inputs`; returning before any load leaves their
# output rows untouched instead of summing that garbage. The constexpr guard
# short-circuits, so num_valid_tokens_ptr is only read when it is provided.
if has_num_valid and pid_m >= tl.load(num_valid_tokens_ptr).to(tl.int32):
return
w_row = topk_weights_ptr + pid_m * top_k
offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)
offs_k = pid_k * BLOCK_K + tl.arange(0, BLOCK_K)
takes = ()
weights = ()
if has_topk_ids:
any_valid = 0
for n in tl.static_range(top_k):
id_val = tl.load(top_ids_ptr + pid_m * top_k + n)
valid = id_val >= 0
any_valid += valid.to(tl.int32)
take = valid
if has_expert_map:
local_id = tl.load(expert_map_ptr + tl.where(valid, id_val, 0))
take = valid & (local_id >= 0)
takes += (take,) # type: ignore[assignment]
weights += (tl.load(w_row + n).to(tl.float32),) # type: ignore[assignment]
# All-padding rows early-return untouched (decode contract past num_recv).
if any_valid == 0:
return
else:
for n in tl.static_range(top_k):
takes += (True,) # type: ignore[assignment]
weights += (tl.load(w_row + n).to(tl.float32),) # type: ignore[assignment]
m_mask = offs_m < num_tokens
k_mask = offs_k < size
mask = m_mask[:, None] & k_mask[None, :]
n_tiles: tl.constexpr = (hidden_size + BLOCK_K - 1) // BLOCK_K
a_row = inputs_ptr + pid_m * stride_m
out_row = outputs_ptr + pid_m * hidden_size
a_base = inputs_ptr + (offs_m * stride_m)[:, None] + offs_k[None, :]
b_base = topk_weights_ptr + offs_m * top_k
acc = tl.zeros((BLOCK_M, BLOCK_K), dtype=tl.float32)
for n in tl.static_range(top_k):
b_val = tl.load(b_base + n, mask=m_mask, other=0.0).to(tl.float32)
if has_expert_map:
id_val = tl.load(top_ids_ptr + offs_m * top_k + n, mask=m_mask, other=0)
expert_mask = tl.load(expert_map_ptr + id_val) >= 0
a_vec = tl.load(
a_base + n * size,
mask=mask & expert_mask[:, None],
other=0.0,
).to(tl.float32)
else:
a_vec = tl.load(
a_base + n * size,
mask=mask,
other=0.0,
).to(tl.float32)
acc += a_vec * b_val[:, None]
out_ptrs = outputs_ptr + (offs_m * size)[:, None] + offs_k[None, :]
tl.store(
out_ptrs,
acc.to(outputs_ptr.dtype.element_ty),
mask=mask,
)
for t in tl.range(0, n_tiles):
offs_k = t * BLOCK_K + tl.arange(0, BLOCK_K)
kmask = offs_k < hidden_size
acc = tl.zeros((BLOCK_K,), dtype=tl.float32)
for n in tl.static_range(top_k):
if takes[n]:
a = tl.load(a_row + n * hidden_size + offs_k, mask=kmask, other=0.0)
acc += a.to(tl.float32) * weights[n]
tl.store(out_row + offs_k, acc.to(outputs_ptr.dtype.element_ty), mask=kmask)
def _heuristic_config(
num_tokens: int,
top_k: int,
size: int,
hidden_size: int,
element_size: int,
):
is_fp32 = element_size > 2
is_sm90_plus = current_platform.has_device_capability(90)
is_sm80_before = not current_platform.has_device_capability(80)
if current_platform.has_device_capability(90):
# SM90/SM100+: prefer small tiles + many CTAs.
if is_fp32:
BLOCK_M = 1 if num_tokens <= 4 else 2
else:
if num_tokens <= 4:
BLOCK_M = 1
elif num_tokens <= 128:
BLOCK_M = 2
else:
BLOCK_M = 4
elif is_fp32:
if num_tokens <= 4:
BLOCK_M = 1
elif num_tokens <= 32:
BLOCK_M = 2
elif num_tokens <= 128:
BLOCK_M = 4
else:
BLOCK_M = 4
else:
if num_tokens <= 4:
BLOCK_M = 1
elif num_tokens <= 32:
BLOCK_M = 2
elif num_tokens <= 128:
BLOCK_M = 4
elif num_tokens <= 1024:
BLOCK_M = 16
else:
BLOCK_M = 8
if is_fp32:
max_block_k = 256
elif is_sm80_before or is_sm90_plus:
max_block_k = 512
else:
max_block_k = 1024
BLOCK_K = min(triton.next_power_of_2(size), max_block_k)
BLOCK_K = max(BLOCK_K, 256)
total = BLOCK_M * BLOCK_K
if is_fp32:
num_warps = max(8, min(16, total // 64))
else:
num_warps = max(4, min(16, total // 256))
if is_sm80_before:
num_warps = min(num_warps, 8)
num_stages = 2
elif is_sm90_plus:
num_warps = min(num_warps, 8)
num_stages = 4 if total <= 2048 else 2
else:
num_stages = 4 if total <= 2048 else 2
return BLOCK_M, BLOCK_K, num_warps, num_stages
max_block_k = 256 if is_fp32 else 512
BLOCK_K = max(128, min(triton.next_power_of_2(hidden_size), max_block_k))
num_warps = 4 if is_fp32 else 2
num_stages = 3
return BLOCK_K, num_warps, num_stages
def moe_fused_mul_sum(
@@ -138,6 +90,7 @@ def moe_fused_mul_sum(
outputs: torch.Tensor | None = None,
topk_ids: torch.Tensor | None = None,
expert_map: torch.Tensor | None = None,
num_valid_tokens: torch.Tensor | None = None,
) -> torch.Tensor:
"""
Fused kernel for MoE (Mixture of Experts) to perform weighted summation
@@ -150,10 +103,21 @@ def moe_fused_mul_sum(
Shape: (num_tokens, top_k).
outputs: Optional pre-allocated output tensor.
Shape: (num_tokens, hidden_size).
topk_ids: Optional indices of the top-k experts. Used when
`expert_map` is provided. Shape: (num_tokens, top_k).
topk_ids: Optional indices of the top-k experts. Shape:
(num_tokens, top_k). A value of -1 marks a slot the expert GEMM
skipped; those slots are excluded from the sum. When provided, rows
with all top ids < 0 (worst-case padding) are skipped and their
output rows left untouched. Required when `expert_map` is provided.
expert_map: Optional mapping for Expert Parallelism. A value < 0
indicates an invalid token/expert pair that will be skipped.
indicates an invalid token/expert pair that will be skipped. Only
needed when `topk_ids` may contain non-local expert ids; if every
non-(-1) id is already a local expert, leave it None to skip the
redundant per-slot lookup.
num_valid_tokens: Optional device scalar (1-element tensor) holding the
number of real token rows (num_recv for a decode dispatch). When
provided, rows past it are left untouched, so the static cudagraph
grid never sums stale padding rows. Pass the token count, not
token*top_k.
Returns:
The fused weighted sum of expert outputs.
@@ -166,34 +130,40 @@ def moe_fused_mul_sum(
assert inputs.dtype in (torch.float32, torch.float16, torch.bfloat16)
assert topk_weights.dtype in (torch.float32, torch.float16, torch.bfloat16)
num_tokens, top_k, size = inputs.shape
output_shape = (num_tokens, size)
num_tokens, top_k, hidden_size = inputs.shape
output_shape = (num_tokens, hidden_size)
if outputs is None:
outputs = torch.empty(output_shape, dtype=inputs.dtype, device=inputs.device)
assert outputs.shape == output_shape
assert topk_weights.shape == (num_tokens, top_k)
assert expert_map is None or topk_ids is not None, (
"topk_ids is required to interpret expert_map"
)
if topk_ids is not None:
assert topk_ids.shape == (num_tokens, top_k)
assert topk_ids.is_contiguous()
assert topk_ids.dtype in (torch.int32, torch.int64)
if not isinstance(inputs, FakeTensor):
BLOCK_M, BLOCK_K, num_warps, num_stages = _heuristic_config(
num_tokens,
top_k,
size,
BLOCK_K, num_warps, num_stages = _heuristic_config(
hidden_size,
inputs.element_size(),
)
grid = (triton.cdiv(size, BLOCK_K), triton.cdiv(num_tokens, BLOCK_M))
grid = (num_tokens,)
moe_fused_mul_sum_kernel[grid](
inputs,
topk_weights,
outputs,
topk_ids,
expert_map,
num_tokens,
top_k * size,
num_valid_tokens,
top_k * hidden_size,
topk_ids is not None,
expert_map is not None,
num_valid_tokens is not None,
top_k,
size,
BLOCK_M,
hidden_size,
BLOCK_K,
num_warps=num_warps,
num_stages=num_stages,
@@ -32,7 +32,7 @@ class DeepEPV2PrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
- Worst-case tensor allocation; padding rows zeroed via
handle.psum_num_recv_tokens_per_scaleup_rank
- Fully cudagraph-capturable
- Expert kernel sorts internally (expert_tokens_meta=None)
- Expert kernel sorts internally (expert_tokens_meta carries no counts)
**Prefill mode (use_cudagraph=False):**
- do_expand=True, do_cpu_sync=True
@@ -266,6 +266,18 @@ class DeepEPV2PrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
if recv_topk_weights is not None and recv_topk_weights.ndim == 1:
recv_topk_weights = recv_topk_weights.unsqueeze(1)
if self.use_cudagraph:
# Carry the per-rank prefix sum so SiTU can skip padding rows.
# expert_num_tokens stays None: count-based consumers (DeepGEMM,
# Triton) must treat a None field as "no counts" and derive their
# own, exactly as in the meta-absent decode case.
if expert_tokens_meta is None:
expert_tokens_meta = mk.ExpertTokensMetadata(
expert_num_tokens=None,
expert_num_tokens_cpu=None,
)
expert_tokens_meta.psum_recv_per_rank = psum_recv_per_rank
if not quant_config.is_block_quantized and not defer_input_quant:
expert_x_scale = None
if expert_x.numel() != 0:
@@ -307,7 +307,18 @@ def _humming_input_schema_to_quant_key(
gs = schema.input_scale_group_size
group_shape = GroupShape(row=1, col=gs) if gs > 0 else GroupShape.PER_TOKEN
scale_dtype = MXFP_SCALE_DTYPE if gs > 0 else torch.float32
# Pick the scale dtype the Humming kernel actually consumes. An explicit
# input_scale_dtype always wins. Otherwise infer from the grouping: MX
# microscale activations (group size 32, e.g. MXFP8) carry an e8m0 (uint8)
# scale, while block-FP8 (group size 128) and per-token FP8/int8 carry a
# float32 scale. Getting this right lets a grouped FP8 activation match
# kFp8Dynamic128Sym instead of an unmatchable uint8-scaled key.
if schema.input_scale_dtype is not None:
scale_dtype = _HUMMING_TO_SCALE_DTYPE[schema.input_scale_dtype]
elif gs == 32:
scale_dtype = MXFP_SCALE_DTYPE
else:
scale_dtype = torch.float32
scale = ScaleDesc(dtype=scale_dtype, static=False, group_shape=group_shape)
@@ -553,6 +564,7 @@ def make_humming_moe_quant_config(
quant_dtype: torch.dtype | str | None,
weight_dtype: torch.dtype | str | None,
weight_group_shape: GroupShape | None = None,
activation_group_shape: GroupShape | None = None,
w1_scale: torch.Tensor | None = None,
w2_scale: torch.Tensor | None = None,
w1_zp: torch.Tensor | None = None,
@@ -569,7 +581,14 @@ def make_humming_moe_quant_config(
assert humming_configs is not None
if quant_dtype is None:
a_quant_desc = FusedMoEQuantDesc(dtype=None)
elif activation_group_shape is not None:
# Pre-dispatch quantization.
a_quant_desc = FusedMoEQuantDesc(
dtype=quant_dtype, shape=activation_group_shape
)
else:
# Deferred path: Humming quantizes the activation internally, so the
# descriptor only needs a non-None dtype to mark it as quantized.
shape = GroupShape(row=1, col=-1)
a_quant_desc = FusedMoEQuantDesc(dtype=quant_dtype, shape=shape)
@@ -621,6 +640,19 @@ def get_humming_moe_quant_config(
else:
q_dtype = str(input_schema.a_dtype)
# Block-FP8 (group-128) activations are quantized *before* the EP all-to-all
# dispatch (so FP8 rather than BF16 crosses the interconnect) and consumed by
# Humming as-is.
activation_group_shape: GroupShape | None = None
input_scale_group_size = getattr(input_schema, "input_scale_group_size", 0) or 0
if (
q_dtype is not None
and q_dtype.startswith("float8")
and input_scale_group_size == 128
):
q_dtype = _HUMMING_TO_QUANT_DTYPE.get(input_schema.a_dtype, FP8_DTYPE)
activation_group_shape = GroupShape(row=1, col=input_scale_group_size)
weight_scale_group_size = weight_schema.weight_scale_group_size
weight_scale_group_size_n = weight_schema.weight_scale_group_size_n
weight_group_shape: tuple[int, ...] = ()
@@ -638,6 +670,7 @@ def get_humming_moe_quant_config(
quant_dtype=q_dtype,
weight_dtype=str(weight_schema.b_dtype),
weight_group_shape=weight_group_shape,
activation_group_shape=activation_group_shape,
w1_scale=getattr(layer, "w13_weight_scale", None),
w1_gscale=getattr(layer, "w13_weight_scale_2", None),
w1_zp=getattr(layer, "w13_zero_point", None),