[Quantization][Humming] Support MXFP4 weight + block-FP8 activation for MoE (#51332)
Signed-off-by: Elvir Crncevic <elvircrn@gmail.com> Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Signed-off-by: Elvir Crnčević <elvircrn@gmail.com> Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com> Co-authored-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
Tyler Michael Smith
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
9c8e90eb26
commit
ca29cd48ba
@@ -201,6 +201,7 @@ def apply_moe_activation(
|
||||
activation_config: ApplyMoEActivationConfig | None = None,
|
||||
topk_ids: torch.Tensor | None = None,
|
||||
expert_map: torch.Tensor | None = None,
|
||||
valid_rows: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Apply MoE activation function.
|
||||
|
||||
@@ -256,6 +257,7 @@ def apply_moe_activation(
|
||||
-1.0
|
||||
if config.activation_situ_linear_beta is None
|
||||
else config.activation_situ_linear_beta,
|
||||
valid_rows,
|
||||
)
|
||||
elif activation == MoEActivation.SWIGLUOAI:
|
||||
torch.ops._C.swigluoai_and_mul(output, input)
|
||||
@@ -286,3 +288,45 @@ def apply_moe_activation(
|
||||
raise ValueError(f"Unsupported FusedMoe activation: {activation}")
|
||||
|
||||
return output
|
||||
|
||||
|
||||
def situ_and_mul_quant(
|
||||
output: torch.Tensor,
|
||||
scale: torch.Tensor,
|
||||
input: torch.Tensor,
|
||||
*,
|
||||
beta: float,
|
||||
linear_beta: float | None,
|
||||
group_size: int = 0,
|
||||
valid_rows: torch.Tensor | None = None,
|
||||
topk: int = 1,
|
||||
) -> None:
|
||||
"""Fused Kimi SITU activation + dynamic FP8 quantization.
|
||||
|
||||
Writes the quantized fp8 down-projection input into ``output`` [M, d] and its
|
||||
float32 scale into ``scale`` (dequant = q * scale), fusing the SITU
|
||||
activation with the FP8 quant that Humming's w2 GEMM would otherwise perform
|
||||
in a separate pass -- this avoids materializing (and rounding through) the
|
||||
bf16 activation_output buffer.
|
||||
|
||||
``group_size`` selects the quant granularity: ``0`` gives per-token scales
|
||||
``scale`` [M, 1]; ``128`` gives k-major block-FP8 group scales ``scale``
|
||||
[M, d // 128], matching humming ``quant_input(group_size=128, float32)``.
|
||||
|
||||
``linear_beta`` <= 0 (or None) means "unset" (up passed through), matching
|
||||
``SituAndMul(linear_beta=None)``. ``valid_rows`` (int32 scalar tensor) is the
|
||||
DeepEP v2 contiguous-layout valid *token* count (e.g. ``psum[-1:]``); the
|
||||
kernel multiplies it by ``topk`` to bound rows, so no host-side cast/multiply
|
||||
is needed. Padding rows are skipped and receive a benign scale of 1.0. Kept
|
||||
in this module so that every ``torch.ops._C.situ*`` call lives in one file.
|
||||
"""
|
||||
torch.ops._C.situ_and_mul_quant(
|
||||
output,
|
||||
scale,
|
||||
input,
|
||||
beta,
|
||||
-1.0 if linear_beta is None else linear_beta,
|
||||
group_size,
|
||||
valid_rows,
|
||||
topk,
|
||||
)
|
||||
|
||||
@@ -525,7 +525,9 @@ def deepgemm_moe_permute(
|
||||
expert_num_tokens = None
|
||||
if expert_tokens_meta is not None:
|
||||
expert_num_tokens = expert_tokens_meta.expert_num_tokens
|
||||
else:
|
||||
if expert_num_tokens is None:
|
||||
# A cudagraph-decode carrier may hold only psum_recv_per_rank and leave
|
||||
# expert_num_tokens None; count locally as in the meta-absent case.
|
||||
expert_num_tokens = count_expert_num_tokens(
|
||||
topk_ids, local_num_experts, expert_map
|
||||
)
|
||||
|
||||
@@ -668,6 +668,7 @@ class NaiveBatchedExperts(mk.FusedMoEExpertsModular):
|
||||
assert hidden_states.dim() == 3
|
||||
assert expert_tokens_meta is not None
|
||||
expert_num_tokens = expert_tokens_meta.expert_num_tokens
|
||||
assert expert_num_tokens is not None
|
||||
|
||||
num_local_experts = w1.size(0)
|
||||
assert num_local_experts == w1.size(0), f"{num_local_experts} == {w1.size(0)}"
|
||||
|
||||
@@ -119,6 +119,37 @@ def get_humming_moe_gemm_type() -> str:
|
||||
return gemm_type
|
||||
|
||||
|
||||
def _fixup_moe_tuning_config(tuning_config: list, max_k_block: int = 128) -> None:
|
||||
"""Fix up each MoE tile in place: cap the K-block and widen warp-N.
|
||||
|
||||
- block_shape[2] (K-block) > ``max_k_block``: the driver rejects the TMA
|
||||
descriptor at launch (CUDA_ERROR_MISALIGNED_ADDRESS). Cap at 128, which
|
||||
Humming already uses for larger M.
|
||||
- warp_shape[1] (warp-N) < 32: block-FP8 (group-128) activations route the
|
||||
w13 (gate/up) GEMM to a tuning table that pins warp-N to 16, under-filling
|
||||
the Hopper WGMMA N dimension and corrupting the GEMM output (gsm8k
|
||||
0.94 -> 0.89). Widen to 32 whenever block_n % 32 == 0 -- on every tile,
|
||||
not just the K-capped ones. w2 (down) already uses warp_n=32.
|
||||
"""
|
||||
logger.info_once("Attempting to override humming GEMM config")
|
||||
for entry in tuning_config:
|
||||
config = entry[2]
|
||||
block_shape = config.get("block_shape")
|
||||
if not (block_shape and len(block_shape) == 3):
|
||||
continue
|
||||
block_m, block_n, block_k = block_shape
|
||||
|
||||
logger.info_once(f"Overriding humming GEMM config. Previous config\n: {config}")
|
||||
if block_k > max_k_block:
|
||||
config["block_shape"] = [block_m, block_n, max_k_block]
|
||||
|
||||
warp_shape = config.get("warp_shape")
|
||||
if warp_shape and warp_shape[1] < 32 and block_n % 32 == 0:
|
||||
config["warp_shape"] = [warp_shape[0], 32, warp_shape[2]]
|
||||
|
||||
logger.info_once(f"Overridden humming GEMM config. Current config\n: {config}")
|
||||
|
||||
|
||||
class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
def __init__(
|
||||
self,
|
||||
@@ -170,6 +201,10 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
gemm_type=self.humming_gemm_type(),
|
||||
)
|
||||
self.compute_config_str = json.dumps(self.compute_config)
|
||||
# Fix up the heuristic-chosen tiles before freezing to JSON -- see
|
||||
# _fixup_moe_tuning_config.
|
||||
_fixup_moe_tuning_config(self.w13_tuning_config)
|
||||
_fixup_moe_tuning_config(self.w2_tuning_config)
|
||||
self.w13_tuning_config_str = json.dumps(self.w13_tuning_config)
|
||||
self.w2_tuning_config_str = json.dumps(self.w2_tuning_config)
|
||||
|
||||
@@ -178,12 +213,17 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
sublayer_name: str,
|
||||
inputs: torch.Tensor,
|
||||
quanted_input: torch.Tensor | None,
|
||||
input_scale: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
from vllm.utils.humming import may_quant_input
|
||||
|
||||
# input_scale is set for block-FP8 (group-128) activations that were
|
||||
# quantized before the EP dispatch: may_quant_input then skips the
|
||||
# redundant w13 quantization and forwards the pre-computed scale.
|
||||
return may_quant_input(
|
||||
self.humming_configs[sublayer_name],
|
||||
inputs=inputs,
|
||||
input_scale=input_scale,
|
||||
quanted_input=quanted_input,
|
||||
)
|
||||
|
||||
@@ -263,6 +303,10 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
(kMxfp4Static, kMxfp4Dynamic),
|
||||
(kMxfp4Static, kMxfp8Dynamic),
|
||||
(kMxfp4Static, kFp8DynamicTokenSym),
|
||||
# MXFP4 weight (group-32 e8m0) with block-FP8 activation
|
||||
# (group-128 float32). Runs via WGMMA software dequant, so it
|
||||
# works on Hopper (SM90/H200) as well as Blackwell.
|
||||
(kMxfp4Static, kFp8Dynamic128Sym),
|
||||
(kNvfp4Static, None),
|
||||
(kNvfp4Static, kFp8DynamicTokenSym),
|
||||
(kMxfp8Static, None),
|
||||
@@ -299,21 +343,43 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
and _is_supported_wna16_weight_key(weight_key)
|
||||
)
|
||||
|
||||
def _prequantizes_dispatch_activation(self) -> bool:
|
||||
"""
|
||||
Whether the prepare/finalize step should quantize activations before
|
||||
the (EP all-to-all) dispatch instead of leaving it to Humming.
|
||||
|
||||
This is enabled only for block-FP8 (group-128) activations: quantizing
|
||||
to FP8 before dispatch sends FP8 rather than BF16 over the interconnect,
|
||||
and Humming then consumes the pre-quantized FP8 + scale directly (see
|
||||
the apply() methods, which forward the dispatch scale into
|
||||
HummingMethod.may_quant_input, and may_quant_input itself, which is a
|
||||
no-op when an input scale is already supplied). The scale layout
|
||||
produced by vLLM's block-FP8 quantization ([M, K // 128] float32,
|
||||
row-major) matches what the Humming WGMMA grouped GEMM expects.
|
||||
"""
|
||||
quant_config = self.quant_config
|
||||
return (
|
||||
quant_config.is_block_quantized
|
||||
and quant_config.quant_dtype == current_platform.fp8_dtype()
|
||||
)
|
||||
|
||||
@property
|
||||
def expects_unquantized_inputs(self) -> bool:
|
||||
"""
|
||||
Humming kernels handle input quantization internally in apply().
|
||||
Whether the prepare/finalize step should defer input quantization to
|
||||
the experts (by setting defer_input_quant=True and passing unquantized
|
||||
inputs).
|
||||
|
||||
This property tells the prepare/finalize step to skip input
|
||||
quantization (by setting defer_input_quant=True) and pass
|
||||
unquantized inputs to the experts. This prevents double
|
||||
quantization: once in prepare and once in Humming's apply().
|
||||
Humming normally quantizes inputs internally via
|
||||
HummingMethod.may_quant_input() in apply(), so we defer quantization
|
||||
(return True) to avoid quantizing twice -- once in prepare and once in
|
||||
Humming's apply().
|
||||
|
||||
Returns:
|
||||
True to indicate that this expert expects unquantized inputs
|
||||
and will handle quantization internally.
|
||||
The exception is block-FP8 (group-128) activations, which are quantized
|
||||
before the dispatch to save interconnect bandwidth (see
|
||||
_prequantizes_dispatch_activation): for those we must NOT defer.
|
||||
"""
|
||||
return True
|
||||
return not self._prequantizes_dispatch_activation()
|
||||
|
||||
@staticmethod
|
||||
def _supports_current_device() -> bool:
|
||||
@@ -585,6 +651,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
activation: MoEActivation,
|
||||
output: torch.Tensor,
|
||||
input: torch.Tensor,
|
||||
valid_rows: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
activation_config = self.activation_config
|
||||
if (
|
||||
@@ -601,8 +668,84 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
activation=activation,
|
||||
input=input,
|
||||
output=output,
|
||||
valid_rows=valid_rows,
|
||||
)
|
||||
|
||||
def fused_situ_quant_enabled(self, activation: MoEActivation) -> bool:
|
||||
"""Whether the SITU activation + w2 quant can be fused into one kernel.
|
||||
|
||||
Fused only for k-major block-FP8 group-128 e4m3 with float32 scales --
|
||||
the sole layout situ_and_mul_quant supports. A float32 as_dtype rules out
|
||||
MXMMA (which uses e8m0 m-major scales), so a group-128 scale is
|
||||
guaranteed k-major. Everything else (m-major MXFP8, other group sizes,
|
||||
16-bit passthrough, non-SITU) falls back to the separate
|
||||
situ_and_mul + quantize_input path.
|
||||
"""
|
||||
if activation != MoEActivation.SITU:
|
||||
return False
|
||||
w2cfg = self.humming_configs["w2"]
|
||||
# Report the specific blocker once so a silent fallback to the unfused
|
||||
# situ_and_mul + quant pair is never a mystery in a trace.
|
||||
reason: str | None = None
|
||||
if not (w2cfg.a_dtype.num_bits == 8 and str(w2cfg.a_dtype) == "float8e4m3"):
|
||||
reason = f"w2 a_dtype is {w2cfg.a_dtype} (need float8e4m3)"
|
||||
elif w2cfg.input_scale_group_size != 128:
|
||||
reason = (
|
||||
f"w2 input_scale_group_size is {w2cfg.input_scale_group_size} "
|
||||
"(need 128)"
|
||||
)
|
||||
elif not (w2cfg.as_dtype is not None and str(w2cfg.as_dtype) == "float32"):
|
||||
reason = f"w2 as_dtype is {w2cfg.as_dtype} (need float32, k-major)"
|
||||
if reason is not None:
|
||||
logger.warning_once(
|
||||
"Humming fused SITU+FP8 quant disabled, using unfused "
|
||||
"situ_and_mul + quant: %s",
|
||||
reason,
|
||||
)
|
||||
return False
|
||||
return True
|
||||
|
||||
def fused_situ_quant(
|
||||
self,
|
||||
gate_up_output: torch.Tensor,
|
||||
quanted_down_input: torch.Tensor,
|
||||
valid_rows: torch.Tensor | None,
|
||||
topk: int = 1,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Run the fused SITU activation + FP8 quant for the w2 input.
|
||||
|
||||
Returns ``(quanted_down_input, input_scale)`` in the same layout the
|
||||
unfused ``apply_activation`` + ``quantize_input("w2")`` pair produces: an
|
||||
fp8 [M, d] tensor plus a k-major block-FP8 float32 [M, d // 128] group
|
||||
scale. Only reached when ``fused_situ_quant_enabled`` (group_size 128).
|
||||
"""
|
||||
from vllm.model_executor.layers.fused_moe.activation import (
|
||||
situ_and_mul_quant,
|
||||
)
|
||||
|
||||
cfg = self.activation_config
|
||||
assert cfg.activation_situ_beta is not None, (
|
||||
"SITU requires activation_situ_beta from FusedMoEConfig"
|
||||
)
|
||||
group_size = self.humming_configs["w2"].input_scale_group_size
|
||||
m, d = quanted_down_input.size(0), quanted_down_input.size(1)
|
||||
input_scale = torch.empty(
|
||||
(m, d // group_size),
|
||||
dtype=torch.float32,
|
||||
device=quanted_down_input.device,
|
||||
)
|
||||
situ_and_mul_quant(
|
||||
quanted_down_input,
|
||||
input_scale,
|
||||
gate_up_output,
|
||||
beta=cfg.activation_situ_beta,
|
||||
linear_beta=cfg.activation_situ_linear_beta,
|
||||
group_size=group_size,
|
||||
valid_rows=valid_rows,
|
||||
topk=topk,
|
||||
)
|
||||
return quanted_down_input, input_scale
|
||||
|
||||
|
||||
class HummingIndexedExperts(HummingExpertsBase):
|
||||
def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
|
||||
@@ -684,10 +827,12 @@ class HummingIndexedExperts(HummingExpertsBase):
|
||||
"""
|
||||
Standard apply implementation for Humming indexed experts.
|
||||
|
||||
Humming performs activation quantization internally and consumes the
|
||||
weights supplied by the modular kernel interface.
|
||||
Results are written to the output buffer supplied by the modular
|
||||
kernel, which may alias workspace13.
|
||||
Note: Humming kernels handle weights internally through the layer
|
||||
object, so w1, w2, a2_scale are unused. a1q_scale is None on the usual
|
||||
path (Humming quantizes the w13 input itself); for block-FP8 activations
|
||||
it carries the scale computed before dispatch, which is forwarded to
|
||||
may_quant_input so Humming skips the redundant w13 quantization. The
|
||||
output is written into workspace13 via the buffer management.
|
||||
"""
|
||||
assert not apply_router_weight_on_input
|
||||
|
||||
@@ -710,6 +855,7 @@ class HummingIndexedExperts(HummingExpertsBase):
|
||||
inputs, input_scale = self.quantize_input(
|
||||
"w13",
|
||||
inputs=hidden_states,
|
||||
input_scale=a1q_scale,
|
||||
quanted_input=buffers.get("quanted_gate_up_input", None),
|
||||
)
|
||||
|
||||
@@ -722,17 +868,45 @@ class HummingIndexedExperts(HummingExpertsBase):
|
||||
**moe_kwargs1,
|
||||
)
|
||||
|
||||
self.apply_activation(
|
||||
activation=activation,
|
||||
input=buffers["gate_up_output"],
|
||||
output=buffers["activation_output"],
|
||||
)
|
||||
# psum[-1:] is the DeepEP valid *token* count as a zero-cost int32 view.
|
||||
# situ (rows = tokens*topk) multiplies by topk on-device; mul_sum bounds
|
||||
# on tokens, so both read this pointer directly -- no host cast/multiply.
|
||||
valid_tokens = None
|
||||
topk = topk_ids.size(1)
|
||||
if (
|
||||
expert_tokens_meta is not None
|
||||
and expert_tokens_meta.psum_recv_per_rank is not None
|
||||
):
|
||||
valid_tokens = expert_tokens_meta.psum_recv_per_rank[-1:]
|
||||
|
||||
inputs, input_scale = self.quantize_input(
|
||||
"w2",
|
||||
inputs=buffers["activation_output"],
|
||||
quanted_input=buffers.get("quanted_down_input", None),
|
||||
)
|
||||
if self.fused_situ_quant_enabled(activation):
|
||||
# Fused SITU + FP8 quant (per-token or block-FP8 group-128) straight
|
||||
# into the w2 input, skipping the bf16 activation_output round-trip.
|
||||
inputs, input_scale = self.fused_situ_quant(
|
||||
gate_up_output=buffers["gate_up_output"],
|
||||
quanted_down_input=buffers["quanted_down_input"],
|
||||
valid_rows=valid_tokens,
|
||||
topk=topk,
|
||||
)
|
||||
else:
|
||||
# Fallback situ_and_mul takes int64 row counts (tokens*topk).
|
||||
valid_rows = (
|
||||
valid_tokens.to(torch.int64) * topk
|
||||
if valid_tokens is not None
|
||||
else None
|
||||
)
|
||||
self.apply_activation(
|
||||
activation=activation,
|
||||
input=buffers["gate_up_output"],
|
||||
output=buffers["activation_output"],
|
||||
valid_rows=valid_rows,
|
||||
)
|
||||
|
||||
inputs, input_scale = self.quantize_input(
|
||||
"w2",
|
||||
inputs=buffers["activation_output"],
|
||||
quanted_input=buffers.get("quanted_down_input", None),
|
||||
)
|
||||
|
||||
self.humming_forward(
|
||||
"w2",
|
||||
@@ -743,12 +917,16 @@ class HummingIndexedExperts(HummingExpertsBase):
|
||||
**moe_kwargs2,
|
||||
)
|
||||
|
||||
# expert_map masks any non-local id; num_valid_tokens bounds the
|
||||
# persistent kernel to the real token rows [0, num_recv) so the padding
|
||||
# tail is never iterated (CUDA-graph-safe device scalar).
|
||||
moe_fused_mul_sum(
|
||||
inputs=buffers["down_output"].view(*topk_ids.shape, -1),
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
expert_map=expert_map,
|
||||
outputs=output,
|
||||
num_valid_tokens=valid_tokens,
|
||||
)
|
||||
|
||||
|
||||
@@ -787,10 +965,13 @@ class HummingGroupedExperts(HummingExpertsBase):
|
||||
"""
|
||||
Standard apply implementation for Humming grouped experts.
|
||||
|
||||
Humming performs activation quantization internally and consumes the
|
||||
weights supplied by the modular kernel interface.
|
||||
Results are written to the output buffer supplied by the modular
|
||||
kernel, which may alias workspace13.
|
||||
Note: Humming kernels handle weights internally through the layer
|
||||
object, so w1, w2, a2_scale are unused. a1q_scale is None on the usual
|
||||
path (Humming quantizes the w13 input itself); for block-FP8 activations
|
||||
it carries the scale computed before dispatch. It is permuted alongside
|
||||
the tokens by moe_permute and forwarded to may_quant_input so Humming
|
||||
skips the redundant w13 quantization. The output is written into
|
||||
workspace13 via the buffer management.
|
||||
"""
|
||||
assert not apply_router_weight_on_input
|
||||
|
||||
@@ -805,9 +986,9 @@ class HummingGroupedExperts(HummingExpertsBase):
|
||||
)
|
||||
buffers["output"] = output
|
||||
|
||||
hidden_states, _, expert_first_token_offset, inv_perm, _ = moe_permute(
|
||||
hidden_states, a1q_scale, expert_first_token_offset, inv_perm, _ = moe_permute(
|
||||
hidden_states=hidden_states,
|
||||
a1q_scale=None,
|
||||
a1q_scale=a1q_scale,
|
||||
topk_ids=topk_ids,
|
||||
n_expert=global_num_experts,
|
||||
n_local_expert=self.num_experts,
|
||||
@@ -818,6 +999,7 @@ class HummingGroupedExperts(HummingExpertsBase):
|
||||
inputs, input_scale = self.quantize_input(
|
||||
"w13",
|
||||
inputs=hidden_states,
|
||||
input_scale=a1q_scale,
|
||||
quanted_input=buffers.get("quanted_gate_up_input", None),
|
||||
)
|
||||
|
||||
@@ -901,15 +1083,21 @@ class BatchedHummingGroupedExperts(HummingExpertsBase):
|
||||
"""
|
||||
Standard apply implementation for Humming batched grouped experts.
|
||||
|
||||
Humming performs activation quantization internally and consumes the
|
||||
weights supplied by the modular kernel interface.
|
||||
Results are written to the output buffer supplied by the modular
|
||||
kernel, which may alias workspace13.
|
||||
Note: Humming kernels handle weights internally through the layer
|
||||
object, so w1, w2, a2_scale are unused. a1q_scale is None on the usual
|
||||
path (Humming quantizes the w13 input itself); for block-FP8 activations
|
||||
it carries the scale computed before dispatch, which is forwarded to
|
||||
may_quant_input so Humming skips the redundant w13 quantization. The
|
||||
output is written into workspace13 via the buffer management.
|
||||
"""
|
||||
assert not apply_router_weight_on_input
|
||||
assert expert_tokens_meta is not None
|
||||
|
||||
hidden_states = hidden_states.view(-1, hidden_states.size(-1))
|
||||
# Keep the (batched) block-FP8 scale row-aligned with the flattened
|
||||
# [num_experts * max_tokens, K] hidden states above.
|
||||
if a1q_scale is not None and a1q_scale.dim() == 3:
|
||||
a1q_scale = a1q_scale.view(-1, a1q_scale.size(-1))
|
||||
valid_shape_m = self.estimate_local_valid_shape_m(topk_ids)
|
||||
expert_num_tokens = expert_tokens_meta.expert_num_tokens
|
||||
|
||||
@@ -925,6 +1113,7 @@ class BatchedHummingGroupedExperts(HummingExpertsBase):
|
||||
inputs, input_scale = self.quantize_input(
|
||||
"w13",
|
||||
inputs=hidden_states,
|
||||
input_scale=a1q_scale,
|
||||
quanted_input=buffers.get("quanted_gate_up_input", None),
|
||||
)
|
||||
|
||||
|
||||
@@ -100,8 +100,9 @@ class ExpertTokensMetadata:
|
||||
Metadata regarding expert-token routing.
|
||||
"""
|
||||
|
||||
expert_num_tokens: torch.Tensor
|
||||
expert_num_tokens: torch.Tensor | None
|
||||
expert_num_tokens_cpu: torch.Tensor | None
|
||||
psum_recv_per_rank: torch.Tensor | None = None
|
||||
|
||||
@staticmethod
|
||||
def make_from_list(
|
||||
@@ -901,6 +902,7 @@ class FusedMoEExpertsModular(FusedMoEExperts):
|
||||
*,
|
||||
topk_ids: torch.Tensor | None = None,
|
||||
expert_map: torch.Tensor | None = None,
|
||||
valid_rows: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
apply_moe_activation(
|
||||
activation,
|
||||
@@ -909,6 +911,7 @@ class FusedMoEExpertsModular(FusedMoEExperts):
|
||||
activation_config=self.activation_config,
|
||||
topk_ids=topk_ids,
|
||||
expert_map=expert_map,
|
||||
valid_rows=valid_rows,
|
||||
)
|
||||
|
||||
@abstractmethod
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
import torch
|
||||
from torch._subclasses.fake_tensor import FakeTensor
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
|
||||
@@ -14,122 +13,75 @@ def moe_fused_mul_sum_kernel(
|
||||
outputs_ptr,
|
||||
top_ids_ptr,
|
||||
expert_map_ptr,
|
||||
num_tokens,
|
||||
num_valid_tokens_ptr,
|
||||
stride_m,
|
||||
has_topk_ids: tl.constexpr,
|
||||
has_expert_map: tl.constexpr,
|
||||
has_num_valid: tl.constexpr,
|
||||
top_k: tl.constexpr,
|
||||
size: tl.constexpr,
|
||||
BLOCK_M: tl.constexpr,
|
||||
hidden_size: tl.constexpr,
|
||||
BLOCK_K: tl.constexpr,
|
||||
):
|
||||
pid_k = tl.program_id(0)
|
||||
pid_m = tl.program_id(1)
|
||||
# One CTA owns one token's output row. Resolve which slots to sum once per
|
||||
# row (the decision is invariant over hidden tiles); the `if take` branch is
|
||||
# block-uniform, so padding slots are skipped rather than masked-and-loaded.
|
||||
pid_m = tl.program_id(0)
|
||||
# Bound to the real recv rows [0, num_recv). Under cudagraph decode the grid
|
||||
# is the static padded token count, and padding rows past num_recv carry
|
||||
# stale ids and stale NaN in `inputs`; returning before any load leaves their
|
||||
# output rows untouched instead of summing that garbage. The constexpr guard
|
||||
# short-circuits, so num_valid_tokens_ptr is only read when it is provided.
|
||||
if has_num_valid and pid_m >= tl.load(num_valid_tokens_ptr).to(tl.int32):
|
||||
return
|
||||
w_row = topk_weights_ptr + pid_m * top_k
|
||||
|
||||
offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)
|
||||
offs_k = pid_k * BLOCK_K + tl.arange(0, BLOCK_K)
|
||||
takes = ()
|
||||
weights = ()
|
||||
if has_topk_ids:
|
||||
any_valid = 0
|
||||
for n in tl.static_range(top_k):
|
||||
id_val = tl.load(top_ids_ptr + pid_m * top_k + n)
|
||||
valid = id_val >= 0
|
||||
any_valid += valid.to(tl.int32)
|
||||
take = valid
|
||||
if has_expert_map:
|
||||
local_id = tl.load(expert_map_ptr + tl.where(valid, id_val, 0))
|
||||
take = valid & (local_id >= 0)
|
||||
takes += (take,) # type: ignore[assignment]
|
||||
weights += (tl.load(w_row + n).to(tl.float32),) # type: ignore[assignment]
|
||||
# All-padding rows early-return untouched (decode contract past num_recv).
|
||||
if any_valid == 0:
|
||||
return
|
||||
else:
|
||||
for n in tl.static_range(top_k):
|
||||
takes += (True,) # type: ignore[assignment]
|
||||
weights += (tl.load(w_row + n).to(tl.float32),) # type: ignore[assignment]
|
||||
|
||||
m_mask = offs_m < num_tokens
|
||||
k_mask = offs_k < size
|
||||
mask = m_mask[:, None] & k_mask[None, :]
|
||||
n_tiles: tl.constexpr = (hidden_size + BLOCK_K - 1) // BLOCK_K
|
||||
a_row = inputs_ptr + pid_m * stride_m
|
||||
out_row = outputs_ptr + pid_m * hidden_size
|
||||
|
||||
a_base = inputs_ptr + (offs_m * stride_m)[:, None] + offs_k[None, :]
|
||||
b_base = topk_weights_ptr + offs_m * top_k
|
||||
|
||||
acc = tl.zeros((BLOCK_M, BLOCK_K), dtype=tl.float32)
|
||||
|
||||
for n in tl.static_range(top_k):
|
||||
b_val = tl.load(b_base + n, mask=m_mask, other=0.0).to(tl.float32)
|
||||
if has_expert_map:
|
||||
id_val = tl.load(top_ids_ptr + offs_m * top_k + n, mask=m_mask, other=0)
|
||||
expert_mask = tl.load(expert_map_ptr + id_val) >= 0
|
||||
a_vec = tl.load(
|
||||
a_base + n * size,
|
||||
mask=mask & expert_mask[:, None],
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
else:
|
||||
a_vec = tl.load(
|
||||
a_base + n * size,
|
||||
mask=mask,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
acc += a_vec * b_val[:, None]
|
||||
|
||||
out_ptrs = outputs_ptr + (offs_m * size)[:, None] + offs_k[None, :]
|
||||
tl.store(
|
||||
out_ptrs,
|
||||
acc.to(outputs_ptr.dtype.element_ty),
|
||||
mask=mask,
|
||||
)
|
||||
for t in tl.range(0, n_tiles):
|
||||
offs_k = t * BLOCK_K + tl.arange(0, BLOCK_K)
|
||||
kmask = offs_k < hidden_size
|
||||
acc = tl.zeros((BLOCK_K,), dtype=tl.float32)
|
||||
for n in tl.static_range(top_k):
|
||||
if takes[n]:
|
||||
a = tl.load(a_row + n * hidden_size + offs_k, mask=kmask, other=0.0)
|
||||
acc += a.to(tl.float32) * weights[n]
|
||||
tl.store(out_row + offs_k, acc.to(outputs_ptr.dtype.element_ty), mask=kmask)
|
||||
|
||||
|
||||
def _heuristic_config(
|
||||
num_tokens: int,
|
||||
top_k: int,
|
||||
size: int,
|
||||
hidden_size: int,
|
||||
element_size: int,
|
||||
):
|
||||
is_fp32 = element_size > 2
|
||||
is_sm90_plus = current_platform.has_device_capability(90)
|
||||
is_sm80_before = not current_platform.has_device_capability(80)
|
||||
|
||||
if current_platform.has_device_capability(90):
|
||||
# SM90/SM100+: prefer small tiles + many CTAs.
|
||||
if is_fp32:
|
||||
BLOCK_M = 1 if num_tokens <= 4 else 2
|
||||
else:
|
||||
if num_tokens <= 4:
|
||||
BLOCK_M = 1
|
||||
elif num_tokens <= 128:
|
||||
BLOCK_M = 2
|
||||
else:
|
||||
BLOCK_M = 4
|
||||
elif is_fp32:
|
||||
if num_tokens <= 4:
|
||||
BLOCK_M = 1
|
||||
elif num_tokens <= 32:
|
||||
BLOCK_M = 2
|
||||
elif num_tokens <= 128:
|
||||
BLOCK_M = 4
|
||||
else:
|
||||
BLOCK_M = 4
|
||||
else:
|
||||
if num_tokens <= 4:
|
||||
BLOCK_M = 1
|
||||
elif num_tokens <= 32:
|
||||
BLOCK_M = 2
|
||||
elif num_tokens <= 128:
|
||||
BLOCK_M = 4
|
||||
elif num_tokens <= 1024:
|
||||
BLOCK_M = 16
|
||||
else:
|
||||
BLOCK_M = 8
|
||||
|
||||
if is_fp32:
|
||||
max_block_k = 256
|
||||
elif is_sm80_before or is_sm90_plus:
|
||||
max_block_k = 512
|
||||
else:
|
||||
max_block_k = 1024
|
||||
BLOCK_K = min(triton.next_power_of_2(size), max_block_k)
|
||||
BLOCK_K = max(BLOCK_K, 256)
|
||||
|
||||
total = BLOCK_M * BLOCK_K
|
||||
if is_fp32:
|
||||
num_warps = max(8, min(16, total // 64))
|
||||
else:
|
||||
num_warps = max(4, min(16, total // 256))
|
||||
|
||||
if is_sm80_before:
|
||||
num_warps = min(num_warps, 8)
|
||||
num_stages = 2
|
||||
elif is_sm90_plus:
|
||||
num_warps = min(num_warps, 8)
|
||||
num_stages = 4 if total <= 2048 else 2
|
||||
else:
|
||||
num_stages = 4 if total <= 2048 else 2
|
||||
|
||||
return BLOCK_M, BLOCK_K, num_warps, num_stages
|
||||
max_block_k = 256 if is_fp32 else 512
|
||||
BLOCK_K = max(128, min(triton.next_power_of_2(hidden_size), max_block_k))
|
||||
num_warps = 4 if is_fp32 else 2
|
||||
num_stages = 3
|
||||
return BLOCK_K, num_warps, num_stages
|
||||
|
||||
|
||||
def moe_fused_mul_sum(
|
||||
@@ -138,6 +90,7 @@ def moe_fused_mul_sum(
|
||||
outputs: torch.Tensor | None = None,
|
||||
topk_ids: torch.Tensor | None = None,
|
||||
expert_map: torch.Tensor | None = None,
|
||||
num_valid_tokens: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Fused kernel for MoE (Mixture of Experts) to perform weighted summation
|
||||
@@ -150,10 +103,21 @@ def moe_fused_mul_sum(
|
||||
Shape: (num_tokens, top_k).
|
||||
outputs: Optional pre-allocated output tensor.
|
||||
Shape: (num_tokens, hidden_size).
|
||||
topk_ids: Optional indices of the top-k experts. Used when
|
||||
`expert_map` is provided. Shape: (num_tokens, top_k).
|
||||
topk_ids: Optional indices of the top-k experts. Shape:
|
||||
(num_tokens, top_k). A value of -1 marks a slot the expert GEMM
|
||||
skipped; those slots are excluded from the sum. When provided, rows
|
||||
with all top ids < 0 (worst-case padding) are skipped and their
|
||||
output rows left untouched. Required when `expert_map` is provided.
|
||||
expert_map: Optional mapping for Expert Parallelism. A value < 0
|
||||
indicates an invalid token/expert pair that will be skipped.
|
||||
indicates an invalid token/expert pair that will be skipped. Only
|
||||
needed when `topk_ids` may contain non-local expert ids; if every
|
||||
non-(-1) id is already a local expert, leave it None to skip the
|
||||
redundant per-slot lookup.
|
||||
num_valid_tokens: Optional device scalar (1-element tensor) holding the
|
||||
number of real token rows (num_recv for a decode dispatch). When
|
||||
provided, rows past it are left untouched, so the static cudagraph
|
||||
grid never sums stale padding rows. Pass the token count, not
|
||||
token*top_k.
|
||||
|
||||
Returns:
|
||||
The fused weighted sum of expert outputs.
|
||||
@@ -166,34 +130,40 @@ def moe_fused_mul_sum(
|
||||
assert inputs.dtype in (torch.float32, torch.float16, torch.bfloat16)
|
||||
assert topk_weights.dtype in (torch.float32, torch.float16, torch.bfloat16)
|
||||
|
||||
num_tokens, top_k, size = inputs.shape
|
||||
output_shape = (num_tokens, size)
|
||||
num_tokens, top_k, hidden_size = inputs.shape
|
||||
output_shape = (num_tokens, hidden_size)
|
||||
if outputs is None:
|
||||
outputs = torch.empty(output_shape, dtype=inputs.dtype, device=inputs.device)
|
||||
|
||||
assert outputs.shape == output_shape
|
||||
assert topk_weights.shape == (num_tokens, top_k)
|
||||
assert expert_map is None or topk_ids is not None, (
|
||||
"topk_ids is required to interpret expert_map"
|
||||
)
|
||||
if topk_ids is not None:
|
||||
assert topk_ids.shape == (num_tokens, top_k)
|
||||
assert topk_ids.is_contiguous()
|
||||
assert topk_ids.dtype in (torch.int32, torch.int64)
|
||||
|
||||
if not isinstance(inputs, FakeTensor):
|
||||
BLOCK_M, BLOCK_K, num_warps, num_stages = _heuristic_config(
|
||||
num_tokens,
|
||||
top_k,
|
||||
size,
|
||||
BLOCK_K, num_warps, num_stages = _heuristic_config(
|
||||
hidden_size,
|
||||
inputs.element_size(),
|
||||
)
|
||||
grid = (triton.cdiv(size, BLOCK_K), triton.cdiv(num_tokens, BLOCK_M))
|
||||
grid = (num_tokens,)
|
||||
moe_fused_mul_sum_kernel[grid](
|
||||
inputs,
|
||||
topk_weights,
|
||||
outputs,
|
||||
topk_ids,
|
||||
expert_map,
|
||||
num_tokens,
|
||||
top_k * size,
|
||||
num_valid_tokens,
|
||||
top_k * hidden_size,
|
||||
topk_ids is not None,
|
||||
expert_map is not None,
|
||||
num_valid_tokens is not None,
|
||||
top_k,
|
||||
size,
|
||||
BLOCK_M,
|
||||
hidden_size,
|
||||
BLOCK_K,
|
||||
num_warps=num_warps,
|
||||
num_stages=num_stages,
|
||||
|
||||
@@ -32,7 +32,7 @@ class DeepEPV2PrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
|
||||
- Worst-case tensor allocation; padding rows zeroed via
|
||||
handle.psum_num_recv_tokens_per_scaleup_rank
|
||||
- Fully cudagraph-capturable
|
||||
- Expert kernel sorts internally (expert_tokens_meta=None)
|
||||
- Expert kernel sorts internally (expert_tokens_meta carries no counts)
|
||||
|
||||
**Prefill mode (use_cudagraph=False):**
|
||||
- do_expand=True, do_cpu_sync=True
|
||||
@@ -266,6 +266,18 @@ class DeepEPV2PrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
|
||||
if recv_topk_weights is not None and recv_topk_weights.ndim == 1:
|
||||
recv_topk_weights = recv_topk_weights.unsqueeze(1)
|
||||
|
||||
if self.use_cudagraph:
|
||||
# Carry the per-rank prefix sum so SiTU can skip padding rows.
|
||||
# expert_num_tokens stays None: count-based consumers (DeepGEMM,
|
||||
# Triton) must treat a None field as "no counts" and derive their
|
||||
# own, exactly as in the meta-absent decode case.
|
||||
if expert_tokens_meta is None:
|
||||
expert_tokens_meta = mk.ExpertTokensMetadata(
|
||||
expert_num_tokens=None,
|
||||
expert_num_tokens_cpu=None,
|
||||
)
|
||||
expert_tokens_meta.psum_recv_per_rank = psum_recv_per_rank
|
||||
|
||||
if not quant_config.is_block_quantized and not defer_input_quant:
|
||||
expert_x_scale = None
|
||||
if expert_x.numel() != 0:
|
||||
|
||||
@@ -307,7 +307,18 @@ def _humming_input_schema_to_quant_key(
|
||||
gs = schema.input_scale_group_size
|
||||
group_shape = GroupShape(row=1, col=gs) if gs > 0 else GroupShape.PER_TOKEN
|
||||
|
||||
scale_dtype = MXFP_SCALE_DTYPE if gs > 0 else torch.float32
|
||||
# Pick the scale dtype the Humming kernel actually consumes. An explicit
|
||||
# input_scale_dtype always wins. Otherwise infer from the grouping: MX
|
||||
# microscale activations (group size 32, e.g. MXFP8) carry an e8m0 (uint8)
|
||||
# scale, while block-FP8 (group size 128) and per-token FP8/int8 carry a
|
||||
# float32 scale. Getting this right lets a grouped FP8 activation match
|
||||
# kFp8Dynamic128Sym instead of an unmatchable uint8-scaled key.
|
||||
if schema.input_scale_dtype is not None:
|
||||
scale_dtype = _HUMMING_TO_SCALE_DTYPE[schema.input_scale_dtype]
|
||||
elif gs == 32:
|
||||
scale_dtype = MXFP_SCALE_DTYPE
|
||||
else:
|
||||
scale_dtype = torch.float32
|
||||
|
||||
scale = ScaleDesc(dtype=scale_dtype, static=False, group_shape=group_shape)
|
||||
|
||||
@@ -553,6 +564,7 @@ def make_humming_moe_quant_config(
|
||||
quant_dtype: torch.dtype | str | None,
|
||||
weight_dtype: torch.dtype | str | None,
|
||||
weight_group_shape: GroupShape | None = None,
|
||||
activation_group_shape: GroupShape | None = None,
|
||||
w1_scale: torch.Tensor | None = None,
|
||||
w2_scale: torch.Tensor | None = None,
|
||||
w1_zp: torch.Tensor | None = None,
|
||||
@@ -569,7 +581,14 @@ def make_humming_moe_quant_config(
|
||||
assert humming_configs is not None
|
||||
if quant_dtype is None:
|
||||
a_quant_desc = FusedMoEQuantDesc(dtype=None)
|
||||
elif activation_group_shape is not None:
|
||||
# Pre-dispatch quantization.
|
||||
a_quant_desc = FusedMoEQuantDesc(
|
||||
dtype=quant_dtype, shape=activation_group_shape
|
||||
)
|
||||
else:
|
||||
# Deferred path: Humming quantizes the activation internally, so the
|
||||
# descriptor only needs a non-None dtype to mark it as quantized.
|
||||
shape = GroupShape(row=1, col=-1)
|
||||
a_quant_desc = FusedMoEQuantDesc(dtype=quant_dtype, shape=shape)
|
||||
|
||||
@@ -621,6 +640,19 @@ def get_humming_moe_quant_config(
|
||||
else:
|
||||
q_dtype = str(input_schema.a_dtype)
|
||||
|
||||
# Block-FP8 (group-128) activations are quantized *before* the EP all-to-all
|
||||
# dispatch (so FP8 rather than BF16 crosses the interconnect) and consumed by
|
||||
# Humming as-is.
|
||||
activation_group_shape: GroupShape | None = None
|
||||
input_scale_group_size = getattr(input_schema, "input_scale_group_size", 0) or 0
|
||||
if (
|
||||
q_dtype is not None
|
||||
and q_dtype.startswith("float8")
|
||||
and input_scale_group_size == 128
|
||||
):
|
||||
q_dtype = _HUMMING_TO_QUANT_DTYPE.get(input_schema.a_dtype, FP8_DTYPE)
|
||||
activation_group_shape = GroupShape(row=1, col=input_scale_group_size)
|
||||
|
||||
weight_scale_group_size = weight_schema.weight_scale_group_size
|
||||
weight_scale_group_size_n = weight_schema.weight_scale_group_size_n
|
||||
weight_group_shape: tuple[int, ...] = ()
|
||||
@@ -638,6 +670,7 @@ def get_humming_moe_quant_config(
|
||||
quant_dtype=q_dtype,
|
||||
weight_dtype=str(weight_schema.b_dtype),
|
||||
weight_group_shape=weight_group_shape,
|
||||
activation_group_shape=activation_group_shape,
|
||||
w1_scale=getattr(layer, "w13_weight_scale", None),
|
||||
w1_gscale=getattr(layer, "w13_weight_scale_2", None),
|
||||
w1_zp=getattr(layer, "w13_zero_point", None),
|
||||
|
||||
Reference in New Issue
Block a user