Merge branch 'main' into patches

Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-16 17:35:42 +07:00
347 changed files with 29531 additions and 3472 deletions
@@ -249,6 +249,8 @@ class FusedMoEQuantConfig:
_w1: FusedMoEQuantDesc
_w2: FusedMoEQuantDesc
is_scale_swizzled: bool = True
# Emit block-quant activation scales column-major (physical [G, M] layout).
is_scale_transposed: bool = False
# MXFP4-specific TRTLLM parameters for SwiGLU activation clamping.
# These correspond to gemm1_alpha, gemm1_beta, gemm1_clamp_limit
@@ -503,6 +505,7 @@ class FusedMoEQuantConfig:
w2_zp: torch.Tensor | None = None,
weight_dtype: torch.dtype | str | None = None,
is_scale_swizzled: bool = True,
is_scale_transposed: bool = False,
gemm1_alpha: float | None = None,
gemm1_beta: float | None = None,
gemm1_clamp_limit: float | None = None,
@@ -537,6 +540,9 @@ class FusedMoEQuantConfig:
- is_scale_swizzled: Whether the activation scale-factor layout is
swizzled. Pass through to the underlying quantization kernel for
dtypes that distinguish layouts (nvfp4, mxfp8). Defaults to True.
- is_scale_transposed: Whether block-quantized activation scales are
emitted column-major (physical [num_groups, num_tokens] layout) by
the quantize step. Defaults to False.
- gemm1_alpha: Optional MXFP4 TRTLLM SwiGLU alpha parameter.
- gemm1_beta: Optional MXFP4 TRTLLM SwiGLU beta parameter.
- gemm1_clamp_limit: Optional MXFP4 TRTLLM SwiGLU clamp limit.
@@ -573,6 +579,7 @@ class FusedMoEQuantConfig:
weight_dtype, w_shape, w2_scale, g2_alphas, w2_zp, w2_bias
),
is_scale_swizzled=is_scale_swizzled,
is_scale_transposed=is_scale_transposed,
gemm1_alpha=gemm1_alpha,
gemm1_beta=gemm1_beta,
gemm1_clamp_limit=gemm1_clamp_limit,
@@ -597,6 +604,7 @@ def fp8_w8a8_moe_quant_config(
a2_gscale: torch.Tensor | None = None,
g1_alphas: torch.Tensor | None = None,
g2_alphas: torch.Tensor | None = None,
is_scale_transposed: bool = False,
gemm1_alpha: float | None = None,
gemm1_beta: float | None = None,
gemm1_clamp_limit: float | None = None,
@@ -617,6 +625,7 @@ def fp8_w8a8_moe_quant_config(
per_act_token_quant=per_act_token_quant,
per_out_ch_quant=per_out_ch_quant,
block_shape=block_shape,
is_scale_transposed=is_scale_transposed,
gemm1_alpha=gemm1_alpha,
gemm1_beta=gemm1_beta,
gemm1_clamp_limit=gemm1_clamp_limit,
File diff suppressed because it is too large Load Diff
@@ -16,6 +16,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kMxfp4Static,
)
from vllm.triton_utils import triton
__all__ = [
"AiterW4A16ExpertsMonolithic",
@@ -52,10 +53,10 @@ def _aiter_w4a16_silu_via_a8w4(
from aiter.ops.triton.moe_op_gemm_a8w4 import moe_gemm_a8w4
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
should_use_cdna4_mx_scale_swizzle,
get_aiter_mx_scale_swizzle,
)
swz = "CDNA4_SCALE" if should_use_cdna4_mx_scale_swizzle() else None
swz = get_aiter_mx_scale_swizzle()
quant_dtype = torch.float8_e4m3fn
g1_gammas = gammas if apply_router_weight_on_input else None
@@ -134,6 +135,8 @@ def aiter_triton_kernel_w4a16_moe_forward(
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
score_mode: str | None = None,
input_ids: torch.Tensor | None = None,
hash_indices_table: torch.Tensor | None = None,
):
assert quant_config is not None and rocm_aiter_ops.is_enabled()
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
@@ -154,17 +157,37 @@ def aiter_triton_kernel_w4a16_moe_forward(
_routing_mod.is_tdm_avail = lambda: False
aiter_routing = _routing_mod.routing
if score_mode is not None:
scale = routed_scaling_factor if routed_scaling_factor is not None else 1.0
if hash_indices_table is not None:
assert score_mode == "sqrtsoftplus" and input_ids is not None
# Same block_m heuristic as aiter's routing().
n_tokens, n_experts = gating_output.shape
block_m = max(
16, min(triton.next_power_of_2(max(1, n_tokens * topk // n_experts)), 128)
)
routing_data, gather_idx, scatter_idx = _routing_mod.routing_from_hash(
gating_output,
hash_indices_table,
input_ids,
topk,
block_m,
score_mode=score_mode,
renorm=renormalize,
routed_scaling_factor=scale,
)
elif score_mode is not None:
use_grouped_topk = num_expert_group is not None and num_expert_group > 1
routing_data, gather_idx, scatter_idx = aiter_routing(
gating_output,
topk,
score_mode=score_mode,
bias=e_score_correction_bias,
renorm=renormalize,
routed_scaling_factor=(
routed_scaling_factor if routed_scaling_factor is not None else 1.0
bias=(
e_score_correction_bias.float()
if e_score_correction_bias is not None
else None
),
renorm=renormalize,
routed_scaling_factor=scale,
use_grouped_topk=use_grouped_topk,
num_expert_group=num_expert_group,
topk_group=topk_group,
@@ -348,6 +371,10 @@ class AiterW4A16ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
RoutingMethodType.DeepSeekV3,
]
@staticmethod
def _supports_hash_routing() -> bool:
return True
@staticmethod
def _supports_router_logits_dtype(
router_logits_dtype: torch.dtype | None,
@@ -375,6 +402,8 @@ class AiterW4A16ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
routing_replay_out: torch.Tensor | None = None,
input_ids: torch.Tensor | None = None,
hash_indices_table: torch.Tensor | None = None,
) -> torch.Tensor:
routing_method = self.moe_config.routing_method
if routing_method == RoutingMethodType.DeepseekV4:
@@ -411,4 +440,6 @@ class AiterW4A16ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
e_score_correction_bias=e_score_correction_bias,
routed_scaling_factor=routed_scaling_factor,
score_mode=score_mode,
input_ids=input_ids,
hash_indices_table=hash_indices_table,
)
@@ -0,0 +1,373 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm._aiter_ops import rocm_aiter_ops
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
FusedMoEQuantConfig,
RoutingMethodType,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kMxfp4Dynamic,
kMxfp4Static,
)
__all__ = [
"AiterW4A4ExpertsMonolithic",
"aiter_triton_kernel_w4a4_moe_forward",
]
def _aiter_raw(t):
if t is None or isinstance(t, torch.Tensor):
return t
return t.storage.data if hasattr(t, "storage") else t
def aiter_triton_kernel_w4a4_moe_forward(
hidden_states: torch.Tensor,
w1,
w2,
gating_output: torch.Tensor,
topk: int,
renormalize: bool,
activation: MoEActivation = MoEActivation.SWIGLUOAI,
quant_config: FusedMoEQuantConfig | None = None,
apply_router_weight_on_input: bool = False,
global_num_experts: int = -1,
expert_map: torch.Tensor | None = None,
unpadded_N_w1=None,
unpadded_K_w1=None,
unpadded_N_w2=None,
unpadded_K_w2=None,
num_expert_group: int | None = None,
topk_group: int | None = None,
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
score_mode: str | None = None,
input_ids: torch.Tensor | None = None,
hash_indices_table: torch.Tensor | None = None,
):
assert quant_config is not None and rocm_aiter_ops.is_enabled()
from aiter.ops.triton.moe.moe_op_gemm_a4w4 import moe_gemm_a4w4, mxfp4_quant
try:
from aiter.ops.triton.moe.moe_routing import routing as _routing_mod
except ImportError:
from aiter.ops.triton.moe_routing import routing as _routing_mod
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
get_aiter_mx_scale_swizzle,
)
from vllm.platforms.rocm import on_gfx1250
if on_gfx1250():
_routing_mod.is_tdm_avail = lambda: False
aiter_routing = _routing_mod.routing
gating_output = torch.nan_to_num(gating_output, nan=0.0, posinf=0.0, neginf=0.0)
if hash_indices_table is not None:
assert input_ids is not None, "hash routing requires input_ids"
n_tokens, n_expts_tot = gating_output.shape
tokens_per_expt = max(1, n_tokens * topk // n_expts_tot)
block_m = max(16, min(1 << (tokens_per_expt - 1).bit_length(), 128))
routing_data, gather_idx, scatter_idx = _routing_mod.routing_from_hash(
gating_output,
hash_indices_table,
input_ids.to(hash_indices_table.dtype),
topk,
block_m,
score_mode=score_mode or "sqrtsoftplus",
renorm=renormalize,
routed_scaling_factor=(
routed_scaling_factor if routed_scaling_factor is not None else 1.0
),
)
elif score_mode is not None:
use_grouped_topk = num_expert_group is not None and num_expert_group > 1
routing_data, gather_idx, scatter_idx = aiter_routing(
gating_output,
topk,
score_mode=score_mode,
bias=(
e_score_correction_bias.float()
if e_score_correction_bias is not None
else None
),
renorm=renormalize,
routed_scaling_factor=(
routed_scaling_factor if routed_scaling_factor is not None else 1.0
),
use_grouped_topk=use_grouped_topk,
num_expert_group=num_expert_group,
topk_group=topk_group,
)
else:
routing_data, gather_idx, scatter_idx = aiter_routing(
gating_output, topk, sm_first=not renormalize
)
if on_gfx1250():
gather_src = gather_idx.to(torch.long) // topk
hidden_states = hidden_states[gather_src]
gather_idx = None
w1_data = _aiter_raw(w1)
w2_data = _aiter_raw(w2)
w1_scale = _aiter_raw(quant_config.w1_scale)
w2_scale = _aiter_raw(quant_config.w2_scale)
gammas = routing_data.gate_scal if routing_data else None
swiglu_limit = (
quant_config.gemm1_clamp_limit
if quant_config.gemm1_clamp_limit is not None
else 7.0
)
# SWIGLUOAI needs the alpha/residual swiglu fused into the GEMM1 epilogue,
# which reads gate/up interleaved along N (see the oracle's weight prep).
fused_swiglu = activation in (
MoEActivation.SWIGLUOAI,
MoEActivation.SWIGLUOAI_UNINTERLEAVE,
)
swiglu_kwargs: dict[str, object] = {}
if fused_swiglu:
assert quant_config.gemm1_beta in (None, 1.0), (
"aiter's fused swiglu hardcodes the residual to (up + 1); "
f"gemm1_beta={quant_config.gemm1_beta} cannot be expressed"
)
swiglu_kwargs = {
"alpha": (
quant_config.gemm1_alpha
if quant_config.gemm1_alpha is not None
else 1.0
),
"limit": swiglu_limit,
"swiglu_add_residual": quant_config.gemm1_beta == 1.0,
}
swizzle_mx_scale = get_aiter_mx_scale_swizzle()
x_q, x_scale = mxfp4_quant(hidden_states.to(torch.bfloat16))
# GEMM1: gate+up projection.
raw_intermediate = moe_gemm_a4w4(
x_q,
w1_data,
x_scale,
w1_scale,
bias=quant_config.w1_bias,
routing_data=routing_data,
gather_indx=gather_idx,
gammas=gammas if apply_router_weight_on_input else None,
swizzle_mx_scale=swizzle_mx_scale,
apply_swiglu=fused_swiglu,
**swiglu_kwargs,
)
if fused_swiglu:
# The swiglu epilogue already halved N.
intermediate = (
raw_intermediate
if unpadded_N_w1 is None
else raw_intermediate[:, : unpadded_N_w1 // 2]
)
else:
if unpadded_N_w1 is not None:
raw_intermediate = raw_intermediate[:, :unpadded_N_w1]
# SiLU(gate) * up on the concatenated [gate | up] halves
from aiter.ops.triton.fusions.fused_clamp_act_mul import fused_clamp_act_mul
half_n = raw_intermediate.shape[-1] // 2
intermediate = torch.empty(
raw_intermediate.shape[0],
half_n,
dtype=raw_intermediate.dtype,
device=raw_intermediate.device,
)
fused_clamp_act_mul(
raw_intermediate,
out=intermediate,
swiglu_limit=swiglu_limit,
activation="silu",
dtype_quant=None,
)
# GEMM2: down projection with scatter-reduce
mid_q, mid_scale = mxfp4_quant(intermediate.to(torch.bfloat16))
out = moe_gemm_a4w4(
mid_q,
w2_data,
mid_scale,
w2_scale,
bias=quant_config.w2_bias,
routing_data=routing_data,
scatter_indx=scatter_idx,
gammas=None if apply_router_weight_on_input else gammas,
swizzle_mx_scale=swizzle_mx_scale,
apply_swiglu=False,
)
return out
class AiterW4A4ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
"""Monolithic MXFP4 W4A4 expert using AITER Triton/Gluon kernels (gfx1250).
Uses moe_gemm_a4w4 (auto-selects gluon backend on gfx1250) with dynamic
MXFP4 activation quantization via mxfp4_quant.
"""
def __init__(
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
):
super().__init__(moe_config, quant_config)
self.topk = moe_config.experts_per_token
self.renormalize = moe_config.routing_method in (
RoutingMethodType.Renormalize,
RoutingMethodType.RenormalizeNaive,
RoutingMethodType.DeepseekV4,
RoutingMethodType.DeepSeekV3,
RoutingMethodType.MiniMax2,
)
@staticmethod
def activation_format() -> mk.FusedMoEActivationFormat:
return mk.FusedMoEActivationFormat.Standard
@staticmethod
def _supports_hash_routing() -> bool:
return True
@staticmethod
def _supports_current_device() -> bool:
if not rocm_aiter_ops.is_enabled():
return False
from vllm.platforms.rocm import on_gfx1250
return on_gfx1250()
@staticmethod
def _supports_no_act_and_mul() -> bool:
return False
@staticmethod
def _supports_quant_scheme(
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
return (weight_key, activation_key) == (kMxfp4Static, kMxfp4Dynamic)
@staticmethod
def _supports_activation(activation: MoEActivation) -> bool:
return activation in (
MoEActivation.SWIGLUOAI,
MoEActivation.SWIGLUOAI_UNINTERLEAVE,
MoEActivation.SILU,
)
@staticmethod
def _supports_parallel_config(
moe_parallel_config: FusedMoEParallelConfig,
) -> bool:
return (
not moe_parallel_config.use_all2all_kernels
and not moe_parallel_config.enable_eplb
and moe_parallel_config.dp_size <= 1
)
@staticmethod
def _supports_routing_method(
routing_method: RoutingMethodType,
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
return routing_method in [
RoutingMethodType.Renormalize,
RoutingMethodType.RenormalizeNaive,
RoutingMethodType.DeepseekV4,
RoutingMethodType.DeepSeekV3,
RoutingMethodType.MiniMax2,
]
@staticmethod
def _supports_router_logits_dtype(
router_logits_dtype: torch.dtype | None,
routing_method: RoutingMethodType,
) -> bool:
return True
@property
def expects_unquantized_inputs(self) -> bool:
return True
def apply(
self,
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
router_logits: torch.Tensor,
activation: MoEActivation,
global_num_experts: int,
expert_map: torch.Tensor | None,
a1q_scale: torch.Tensor | None,
apply_router_weight_on_input: bool,
num_expert_group: int | None = None,
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
routing_replay_out: torch.Tensor | None = None,
input_ids: torch.Tensor | None = None,
hash_indices_table: torch.Tensor | None = None,
) -> torch.Tensor:
assert self.moe_config.intermediate_size_per_partition_unpadded is not None
assert self.moe_config.hidden_dim_unpadded is not None
routing_method = self.moe_config.routing_method
if routing_method == RoutingMethodType.DeepseekV4:
score_mode = "sqrtsoftplus"
elif routing_method in (
RoutingMethodType.DeepSeekV3,
RoutingMethodType.MiniMax2,
):
# Sigmoid scores + routing bias; grouped for DeepSeek-R1/V3,
# ungrouped for MiniMax-M3.
score_mode = "sigmoid"
else:
score_mode = None
return aiter_triton_kernel_w4a4_moe_forward(
hidden_states=hidden_states,
w1=w1,
w2=w2,
gating_output=router_logits,
topk=self.topk,
renormalize=self.renormalize,
activation=activation,
global_num_experts=global_num_experts,
expert_map=expert_map,
quant_config=self.quant_config,
apply_router_weight_on_input=apply_router_weight_on_input,
unpadded_N_w1=(
self.moe_config.intermediate_size_per_partition_unpadded * 2
),
unpadded_K_w1=self.moe_config.hidden_dim_unpadded,
unpadded_N_w2=self.moe_config.hidden_dim_unpadded,
unpadded_K_w2=(self.moe_config.intermediate_size_per_partition_unpadded),
num_expert_group=num_expert_group,
topk_group=topk_group,
e_score_correction_bias=e_score_correction_bias,
routed_scaling_factor=routed_scaling_factor,
score_mode=score_mode,
input_ids=input_ids,
hash_indices_table=hash_indices_table,
)
@@ -61,12 +61,6 @@ def aiter_triton_kernel_w4a8_moe_forward(
gating_output, topk, sm_first=not renormalize
)
# gfx1250: aiter's in-kernel gather is numerically broken
if on_gfx1250():
gather_src = gather_idx.to(torch.long) // topk
hidden_states = hidden_states[gather_src]
gather_idx = None
return triton_kernel_fused_mxfp4_w4a8_experts(
None,
hidden_states,
@@ -130,11 +124,11 @@ def triton_kernel_fused_mxfp4_w4a8_experts(
from aiter.ops.triton.quant_moe import downcast_to_static_fp8
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
should_use_cdna4_mx_scale_swizzle,
get_aiter_mx_scale_swizzle,
weight_mx_scale,
)
_swizzle_mx_scale = "CDNA4_SCALE" if should_use_cdna4_mx_scale_swizzle() else None
_swizzle_mx_scale = get_aiter_mx_scale_swizzle()
assert quant_config.w1_precision is not None, (
"w1_precision in quant config can't be None"
@@ -436,6 +436,7 @@ def FusedMoEFactory(
# TODO get from router? needs to be truncated?
e_score_correction_bias=e_score_correction_bias,
apply_router_weight_on_input=apply_router_weight_on_input,
hash_indices_table=hash_indices_table,
**routed_experts_args if routed_experts_args is not None else {},
)
@@ -5,7 +5,7 @@ from collections.abc import Callable
from dataclasses import dataclass
from enum import Enum
from math import prod
from typing import final
from typing import Any, final
import torch
@@ -579,7 +579,11 @@ class FusedMoEExperts(ABC):
return False, _make_reason(
f"parallel config {moe_config.moe_parallel_config}"
)
elif moe_config.has_hash_routing and cls.is_monolithic():
elif (
moe_config.has_hash_routing
and cls.is_monolithic()
and not cls._supports_hash_routing()
):
return False, _make_reason("hash routing")
elif moe_config.require_decomposed_backend and cls.is_monolithic():
return False, _make_reason("routes adjusted after routing")
@@ -659,6 +663,15 @@ class FusedMoEExperts(ABC):
"""
return True
@staticmethod
def _supports_hash_routing() -> bool:
"""Whether a monolithic kernel implements hash routing itself.
Kernels that return True receive `input_ids` and `hash_indices_table`
in `apply()` for layers with a hash-routing table.
"""
return False
@staticmethod
def _supports_router_logits_dtype(
router_logits_dtype: torch.dtype | None,
@@ -1584,11 +1597,20 @@ class FusedMoEKernelMonolithicImpl:
topk_group: int | None = None,
*,
routing_sink: RoutedExpertsSink | None,
input_ids: torch.Tensor | None = None,
hash_indices_table: torch.Tensor | None = None,
) -> torch.Tensor | UnfinalizedMoEOutput:
"""Same as forward(), except uses router_logits as opposed
to the topk_ids and topk_weights. This is used for kernels
that have fused router + experts (e.g. FLASHINFER_TRTLLM).
"""
# Only experts with _supports_hash_routing() accept these.
hash_kwargs: dict[str, Any] = {}
if hash_indices_table is not None:
hash_kwargs = {
"input_ids": input_ids,
"hash_indices_table": hash_indices_table,
}
a1q, a1q_scale, router_logits = self.prepare_finalize.prepare(
hidden_states,
router_logits=router_logits,
@@ -1605,6 +1627,11 @@ class FusedMoEKernelMonolithicImpl:
f"kernel {type(self.fused_experts).__name__}."
)
routing_replay_out = routing_sink.buffer[: len(a1q)]
hash_kwargs = (
{"input_ids": input_ids, "hash_indices_table": hash_indices_table}
if hash_indices_table is not None
else {}
)
fused_out = self.fused_experts.apply(
hidden_states=a1q,
w1=w1,
@@ -1621,6 +1648,7 @@ class FusedMoEKernelMonolithicImpl:
routed_scaling_factor=routed_scaling_factor,
topk_group=topk_group,
routing_replay_out=routing_replay_out,
**hash_kwargs,
)
if routing_sink is not None:
routing_sink.capture_fn(routing_replay_out)
@@ -1733,6 +1761,8 @@ class FusedMoEKernel:
topk_group: int | None = None,
*,
routing_sink: RoutedExpertsSink | None,
input_ids: torch.Tensor | None = None,
hash_indices_table: torch.Tensor | None = None,
) -> torch.Tensor | UnfinalizedMoEOutput:
assert isinstance(self.impl, FusedMoEKernelMonolithicImpl)
return self.impl.apply(
@@ -1749,6 +1779,8 @@ class FusedMoEKernel:
routed_scaling_factor=routed_scaling_factor,
topk_group=topk_group,
routing_sink=routing_sink,
input_ids=input_ids,
hash_indices_table=hash_indices_table,
)
def apply(
@@ -841,6 +841,11 @@ def make_fp8_moe_quant_config(
block_shape=block_shape,
per_act_token_quant=per_act_token_quant,
per_out_ch_quant=per_out_ch_quant,
# TRTLLM fp8-block consumes activation scales as [K/128, M]; emit them
# transposed from the quant kernel instead of copying afterwards.
is_scale_transposed=(
fp8_backend == Fp8MoeBackend.FLASHINFER_TRTLLM and block_shape == [128, 128]
),
gemm1_alpha=gemm1_alpha,
gemm1_beta=gemm1_beta,
gemm1_clamp_limit=swiglu_limit,
@@ -152,7 +152,9 @@ class Mxfp4MoeBackend(Enum):
# W4A16: aiter Triton moe_gemm_a16w4 kernel (gfx942/gfx950/gfx1250)
AITER_TRITON_MXFP4_BF16 = "AITER_TRITON_MXFP4_BF16"
AITER_MXFP4_FP8 = "AITER_MXFP4_FP8" # W4A8: triton kernel
AITER_MXFP4_MXFP4 = "AITER_MXFP4_MXFP4" # W4A4: CK kernel
AITER_MXFP4_MXFP4 = "AITER_MXFP4_MXFP4" # W4A4: CK kernel (gfx950)
# W4A4: aiter Triton/Gluon moe_gemm_a4w4 kernel (gfx1250)
AITER_TRITON_MXFP4_MXFP4 = "AITER_TRITON_MXFP4_MXFP4"
# Triton
TRITON = "TRITON"
TRITON_UNFUSED = "TRITON_UNFUSED"
@@ -310,6 +312,13 @@ def backend_to_kernel_cls(
return [AiterMxfp4Experts]
elif backend == Mxfp4MoeBackend.AITER_TRITON_MXFP4_MXFP4:
from vllm.model_executor.layers.fused_moe.experts.aiter_mxfp4_w4a4_moe import (
AiterW4A4ExpertsMonolithic,
)
return [AiterW4A4ExpertsMonolithic]
elif backend == Mxfp4MoeBackend.XPU:
from vllm.model_executor.layers.fused_moe.experts.xpu_moe import XPUExpertsMxFp4
@@ -370,9 +379,11 @@ def map_mxfp4_backend(runner_backend: MoEBackend) -> list[Mxfp4MoeBackend]:
Mxfp4MoeBackend.AITER_TRITON_MXFP4_BF16,
Mxfp4MoeBackend.AITER_MXFP4_FP8,
Mxfp4MoeBackend.AITER_MXFP4_MXFP4,
Mxfp4MoeBackend.AITER_TRITON_MXFP4_MXFP4,
],
"aiter_triton_mxfp4_bf16": [Mxfp4MoeBackend.AITER_TRITON_MXFP4_BF16],
"aiter_mxfp4_fp8": [Mxfp4MoeBackend.AITER_MXFP4_FP8],
"aiter_triton_mxfp4_mxfp4": [Mxfp4MoeBackend.AITER_TRITON_MXFP4_MXFP4],
"aiter_mxfp4_mxfp4": [Mxfp4MoeBackend.AITER_MXFP4_MXFP4],
"xpu": [Mxfp4MoeBackend.XPU],
"cpu": [Mxfp4MoeBackend.CPU],
@@ -464,6 +475,7 @@ def _backend_activation_key(backend: Mxfp4MoeBackend) -> QuantKey | None:
return kFp8StaticTensorSym
if backend in (
Mxfp4MoeBackend.AITER_MXFP4_MXFP4,
Mxfp4MoeBackend.AITER_TRITON_MXFP4_MXFP4,
Mxfp4MoeBackend.CUTLASS_MXFP4_MXFP4,
):
return kMxfp4Dynamic
@@ -896,6 +908,18 @@ def mxfp4_round_up_hidden_size_and_intermediate_size(
return hidden_size, intermediate_size
def _interleave_gate_up_rows(t: torch.Tensor) -> torch.Tensor:
"""Repack [E, 2n, X] from [all gates | all ups] to [g0, u0, g1, u1, ...]."""
num_experts, two_n, inner = t.shape
assert two_n % 2 == 0, f"expected an even gate/up row count, got {two_n}"
return (
t.view(num_experts, 2, two_n // 2, inner)
.transpose(1, 2)
.contiguous()
.view(num_experts, two_n, inner)
)
def convert_gpt_oss_weight_to_mxfp4_moe_kernel_format(
mxfp4_backend: Mxfp4MoeBackend,
layer: torch.nn.Module,
@@ -1192,9 +1216,56 @@ def convert_gpt_oss_weight_to_mxfp4_moe_kernel_format(
w2_bias,
)
elif mxfp4_backend == Mxfp4MoeBackend.AITER_MXFP4_MXFP4:
from vllm._aiter_ops import rocm_aiter_ops
elif mxfp4_backend == Mxfp4MoeBackend.AITER_TRITON_MXFP4_MXFP4:
if w13_bias is not None:
w13_bias = w13_bias.data.to(torch.float32)
if w2_bias is not None:
w2_bias = w2_bias.data.to(torch.float32)
# Triton moe_gemm_a4w4: uint8 column-major weights.
w13_data = w13_weight.data.view(torch.uint8)
w2_data = w2_weight.data.view(torch.uint8)
w13_scale_data = w13_weight_scale.data.view(torch.uint8)
# SILU applies the activation outside the GEMM and keeps the
# checkpoint's packed halves; the SWIGLUOAI epilogue needs them
# interleaved.
if getattr(layer, "activation", None) in (
MoEActivation.SWIGLUOAI,
MoEActivation.SWIGLUOAI_UNINTERLEAVE,
):
w13_data = _interleave_gate_up_rows(w13_data)
w13_scale_data = _interleave_gate_up_rows(w13_scale_data)
# Transpose to [E, K, N] — the view has stride(-2)==1 (column-major),
# which is what moe_gemm_a4w4 asserts.
w13_data = w13_data.transpose(1, 2)
w2_data = w2_data.transpose(1, 2)
# Scales are [E, N, K_scale]; both kernels index them as
# [E, K_scale, N] with K innermost, so transpose(1, 2)
# Keep them uint8: Triton moe_gemm_a4w4 takes e8m0 scales
w13_scale = w13_scale_data.transpose(1, 2)
w2_scale = w2_weight_scale.data.view(torch.uint8).transpose(1, 2)
from aiter.ops.triton.utils.shuffle import shuffle_scale_moe
w13_scale = shuffle_scale_moe(
w13_scale,
arch="gfx1250",
preshuffle_factor=32,
scale_kwidth=4,
)
w2_scale = shuffle_scale_moe(
w2_scale,
arch="gfx1250",
preshuffle_factor=32,
scale_kwidth=4,
)
return (w13_data, w2_data, w13_scale, w2_scale, w13_bias, w2_bias)
elif mxfp4_backend == Mxfp4MoeBackend.AITER_MXFP4_MXFP4:
if w13_bias is not None:
w13_bias = w13_bias.data.to(torch.float32)
if w2_bias is not None:
@@ -1203,6 +1274,8 @@ def convert_gpt_oss_weight_to_mxfp4_moe_kernel_format(
# e8m0_shuffle on weight scales (GFX950 swizzle layout)
from aiter.utility.fp4_utils import e8m0_shuffle
from vllm._aiter_ops import rocm_aiter_ops
s0, s1, _ = w13_weight_scale.shape
w13_weight_scale.data = e8m0_shuffle(w13_weight_scale.view(s0 * s1, -1)).view(
s0, s1, -1
@@ -1935,6 +2008,7 @@ def convert_weight_to_mxfp4_moe_kernel_format(
)
elif mxfp4_backend in (
Mxfp4MoeBackend.AITER_MXFP4_MXFP4,
Mxfp4MoeBackend.AITER_TRITON_MXFP4_MXFP4,
Mxfp4MoeBackend.AITER_MXFP4_FP8,
):
return convert_gpt_oss_weight_to_mxfp4_moe_kernel_format(
@@ -2088,6 +2162,7 @@ def make_mxfp4_moe_quant_config(
)
elif mxfp4_backend in (
Mxfp4MoeBackend.AITER_MXFP4_MXFP4,
Mxfp4MoeBackend.AITER_TRITON_MXFP4_MXFP4,
Mxfp4MoeBackend.CUTLASS_MXFP4_MXFP4,
):
return ocp_mx_moe_quant_config(
@@ -31,6 +31,7 @@ def _quantize_input(
per_act_token_quant=quant_config.per_act_token_quant,
block_shape=quant_config.block_shape,
is_scale_swizzled=quant_config.is_scale_swizzled,
is_scale_transposed=quant_config.is_scale_transposed,
mx_alignment=quant_config.mx_alignment,
)
@@ -0,0 +1,682 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Triton input-staging kernels for DeepGEMM MegaMoE.
Quantize hidden states to fp8 with E8M0 group scales, to NVFP4 with per-token
global scales, or copy them as BF16, and repack the routing top-k tensors into
the int64/float32 layout that the DeepGEMM MegaMoE kernels consume.
"""
import torch
from vllm.model_executor.layers.quantization.utils.quant_utils import (
FP8_DTYPE,
QuantKey,
kMxfp8Dynamic,
)
from vllm.triton_utils import tl, triton
_E8M0_SCALE_DTYPES = (torch.uint8, torch.float8_e8m0fnu)
# Inverse of the NVFP4 range: E2M1 max (6) times E4M3 max (448).
_INV_NVFP4_MAX = tl.constexpr(1.0 / 2688.0)
@triton.jit
def _quantize_fp8_groups(
hidden,
BLOCK_M: tl.constexpr,
BLOCK_K: tl.constexpr,
GROUP_K: tl.constexpr,
USE_UE8M0: tl.constexpr,
):
"""Return fp8 values, fp32 group scales and, for UE8M0, the scale exponents."""
num_groups: tl.constexpr = BLOCK_K // GROUP_K
hidden_groups = tl.reshape(tl.abs(hidden), [BLOCK_M, num_groups, GROUP_K])
amax = tl.max(hidden_groups, axis=2)
amax = tl.maximum(amax, 1.0e-4)
scale = amax / 448.0
if USE_UE8M0:
scale_bits = scale.to(tl.uint32, bitcast=True)
scale_exp = ((scale_bits >> 23) & 0xFF) + ((scale_bits & 0x7FFFFF) != 0).to(
tl.uint32
)
scale_exp = tl.minimum(tl.maximum(scale_exp, 1), 254)
quant_scale = (scale_exp << 23).to(tl.float32, bitcast=True)
else:
scale_exp = scale.to(tl.uint32, bitcast=True)
quant_scale = scale
hidden_groups = tl.reshape(hidden, [BLOCK_M, num_groups, GROUP_K])
scaled = hidden_groups * (1.0 / quant_scale)[:, :, None]
scaled = tl.reshape(scaled, [BLOCK_M, BLOCK_K])
return scaled.to(tl.float8e4nv), quant_scale, scale_exp
@triton.jit
def _pack_bytes(values, NUM_BYTES: tl.constexpr):
"""Pack the last dim of ``[..., NUM_BYTES]`` byte values into int32 words."""
offsets = tl.arange(0, NUM_BYTES)
return tl.sum(values.to(tl.uint32) << (offsets * 8), axis=-1).to(tl.int32)
@triton.jit
def _shared_sf_row(token_id, SHARED_BLOCK_M: tl.constexpr):
"""Row of ``token_id`` in the shared-expert MN-major UTCCP scale pages."""
m_block_id = token_id // SHARED_BLOCK_M
m_in_block = token_id % SHARED_BLOCK_M
aligned_block_m: tl.constexpr = triton.cdiv(SHARED_BLOCK_M, 128) * 128
transposed_m = (
(m_in_block // 128) * 128 + (m_in_block % 32) * 4 + (m_in_block % 128) // 32
)
return m_block_id * aligned_block_m + transposed_m
@triton.jit
def _store_topk(
token_id,
token_mask,
topk_ids,
topk_weights,
is_padding,
topk_idx_out,
topk_weights_out,
topk_ids_stride_m: tl.constexpr,
topk_ids_stride_k: tl.constexpr,
topk_weights_stride_m: tl.constexpr,
topk_weights_stride_k: tl.constexpr,
is_padding_stride_m: tl.constexpr,
topk_idx_stride_m: tl.constexpr,
topk_idx_stride_k: tl.constexpr,
topk_weights_out_stride_m: tl.constexpr,
topk_weights_out_stride_k: tl.constexpr,
top_k: tl.constexpr,
BLOCK_M: tl.constexpr,
BLOCK_TOPK: tl.constexpr,
) -> None:
"""Repack top-k routing as int64 ids and fp32 weights; padding is unrouted."""
topk_offsets = tl.arange(0, BLOCK_TOPK)
topk_mask = token_mask[:, None] & (topk_offsets[None, :] < top_k)
token_is_padding = tl.full((BLOCK_M,), False, tl.int1)
if is_padding is not None:
token_is_padding = tl.load(
is_padding + token_id * is_padding_stride_m,
mask=token_mask,
other=True,
)
ids = tl.load(
topk_ids
+ token_id[:, None] * topk_ids_stride_m
+ topk_offsets[None, :] * topk_ids_stride_k,
mask=topk_mask,
other=0,
).to(tl.int64)
ids = tl.where(token_is_padding[:, None], -1, ids)
tl.store(
topk_idx_out
+ token_id[:, None] * topk_idx_stride_m
+ topk_offsets[None, :] * topk_idx_stride_k,
ids,
mask=topk_mask,
)
weights = tl.load(
topk_weights
+ token_id[:, None] * topk_weights_stride_m
+ topk_offsets[None, :] * topk_weights_stride_k,
mask=topk_mask,
other=0.0,
)
weights = tl.where(token_is_padding[:, None], 0.0, weights)
tl.store(
topk_weights_out
+ token_id[:, None] * topk_weights_out_stride_m
+ topk_offsets[None, :] * topk_weights_out_stride_k,
weights,
mask=topk_mask,
)
@triton.jit(do_not_specialize=["num_tokens"])
def _prepare_megamoe_inputs_kernel(
hidden_states,
x_out,
x_sf,
shared_x_sf,
topk_ids,
topk_weights,
is_padding,
topk_idx_out,
topk_weights_out,
hidden_stride_m: tl.constexpr,
hidden_stride_k: tl.constexpr,
x_stride_m: tl.constexpr,
x_stride_k: tl.constexpr,
x_sf_stride_m: tl.constexpr,
x_sf_stride_k: tl.constexpr,
shared_x_sf_stride_m: tl.constexpr,
shared_x_sf_stride_k: tl.constexpr,
topk_ids_stride_m: tl.constexpr,
topk_ids_stride_k: tl.constexpr,
topk_weights_stride_m: tl.constexpr,
topk_weights_stride_k: tl.constexpr,
is_padding_stride_m: tl.constexpr,
topk_idx_stride_m: tl.constexpr,
topk_idx_stride_k: tl.constexpr,
topk_weights_out_stride_m: tl.constexpr,
topk_weights_out_stride_k: tl.constexpr,
num_tokens,
hidden_size: tl.constexpr,
top_k: tl.constexpr,
BLOCK_M: tl.constexpr,
BLOCK_K: tl.constexpr,
GROUP_K: tl.constexpr,
BLOCK_TOPK: tl.constexpr,
SHARED_BLOCK_M: tl.constexpr,
QUANTIZE: tl.constexpr,
USE_UE8M0: tl.constexpr,
) -> None:
token_id = tl.program_id(0) * BLOCK_M + tl.arange(0, BLOCK_M)
token_mask = token_id < num_tokens
k_block_id = tl.program_id(1)
k_offsets = k_block_id * BLOCK_K + tl.arange(0, BLOCK_K)
k_mask = token_mask[:, None] & (k_offsets[None, :] < hidden_size)
hidden = tl.load(
hidden_states
+ token_id[:, None] * hidden_stride_m
+ k_offsets[None, :] * hidden_stride_k,
mask=k_mask,
other=0.0,
).to(tl.float32)
if not QUANTIZE:
tl.store(
x_out + token_id[:, None] * x_stride_m + k_offsets[None, :] * x_stride_k,
hidden.to(x_out.dtype.element_ty),
mask=k_mask,
)
else:
num_groups: tl.constexpr = BLOCK_K // GROUP_K
fp8, quant_scale, scale_exp = _quantize_fp8_groups(
hidden, BLOCK_M, BLOCK_K, GROUP_K, USE_UE8M0
)
tl.store(
x_out + token_id[:, None] * x_stride_m + k_offsets[None, :] * x_stride_k,
fp8,
mask=k_mask,
)
if USE_UE8M0:
packed_scale = _pack_bytes(scale_exp, num_groups)
tl.store(
x_sf + token_id * x_sf_stride_m + k_block_id * x_sf_stride_k,
packed_scale,
mask=token_mask,
)
else:
sf_offset = tl.arange(0, num_groups)
tl.store(
x_sf
+ token_id[:, None] * x_sf_stride_m
+ k_block_id * x_sf_stride_k
+ sf_offset[None, :],
quant_scale,
mask=token_mask[:, None],
)
# DeepGEMM's SM100 shared-expert TMA loads require the activation scales
# in an MN-major layout whose row permutation depends on the MegaMoE
# scheduler's runtime BLOCK_M. Write that view while the packed UE8M0 scale
# is already resident, avoiding another kernel and temporary tensor.
if USE_UE8M0 and shared_x_sf is not None:
shared_row = _shared_sf_row(token_id, SHARED_BLOCK_M)
tl.store(
shared_x_sf
+ shared_row * shared_x_sf_stride_m
+ k_block_id * shared_x_sf_stride_k,
packed_scale,
mask=token_mask,
)
if k_block_id == 0:
_store_topk(
token_id,
token_mask,
topk_ids,
topk_weights,
is_padding,
topk_idx_out,
topk_weights_out,
topk_ids_stride_m,
topk_ids_stride_k,
topk_weights_stride_m,
topk_weights_stride_k,
is_padding_stride_m,
topk_idx_stride_m,
topk_idx_stride_k,
topk_weights_out_stride_m,
topk_weights_out_stride_k,
top_k,
BLOCK_M,
BLOCK_TOPK,
)
@triton.jit(do_not_specialize=["num_tokens", "l2_activation_scale"])
def _prepare_megamoe_nvfp4_inputs_kernel(
hidden_states,
x_out,
x_sf,
x_scales,
shared_x,
shared_x_sf,
topk_ids,
topk_weights,
is_padding,
topk_idx_out,
topk_weights_out,
w2_global_scale,
l2_alpha,
hidden_stride_m: tl.constexpr,
hidden_stride_k: tl.constexpr,
x_stride_m: tl.constexpr,
x_sf_stride_m: tl.constexpr,
shared_x_stride_m: tl.constexpr,
shared_x_sf_stride_m: tl.constexpr,
shared_x_sf_stride_k: tl.constexpr,
topk_ids_stride_m: tl.constexpr,
topk_ids_stride_k: tl.constexpr,
topk_weights_stride_m: tl.constexpr,
topk_weights_stride_k: tl.constexpr,
is_padding_stride_m: tl.constexpr,
topk_idx_stride_m: tl.constexpr,
topk_idx_stride_k: tl.constexpr,
topk_weights_out_stride_m: tl.constexpr,
topk_weights_out_stride_k: tl.constexpr,
num_tokens,
l2_activation_scale,
hidden_size: tl.constexpr,
top_k: tl.constexpr,
num_local_experts: tl.constexpr,
BLOCK_M: tl.constexpr,
BLOCK_K: tl.constexpr,
BLOCK_TOPK: tl.constexpr,
BLOCK_E: tl.constexpr,
SHARED_BLOCK_M: tl.constexpr,
SHARED_MXFP8: tl.constexpr,
) -> None:
if l2_alpha is not None:
experts = tl.arange(0, BLOCK_E)
expert_mask = (experts < num_local_experts) & (tl.program_id(0) == 0)
alpha = tl.load(w2_global_scale + experts, mask=expert_mask)
tl.store(l2_alpha + experts, alpha * l2_activation_scale, mask=expert_mask)
token_id = tl.program_id(0) * BLOCK_M + tl.arange(0, BLOCK_M)
token_mask = token_id < num_tokens
k_offsets = tl.arange(0, BLOCK_K)
hidden_ptrs = hidden_states + token_id[:, None] * hidden_stride_m
amax = tl.zeros((BLOCK_M,), tl.float32)
for k_block_id in range(hidden_size // BLOCK_K):
hidden = tl.load(
hidden_ptrs + (k_block_id * BLOCK_K + k_offsets[None, :]) * hidden_stride_k,
mask=token_mask[:, None],
other=0.0,
).to(tl.float32)
amax = tl.maximum(amax, tl.max(tl.abs(hidden), axis=1))
x_scale = tl.maximum(amax, 1.0e-12) * _INV_NVFP4_MAX
tl.store(x_scales + token_id, x_scale, mask=token_mask)
for k_block_id in range(hidden_size // BLOCK_K):
k_ids = k_block_id * BLOCK_K + k_offsets
hidden = tl.load(
hidden_ptrs + k_ids[None, :] * hidden_stride_k,
mask=token_mask[:, None],
other=0.0,
)
# E2M1 values with one E4M3 scale per 16, relative to the token scale.
values = tl.reshape(
tl.div_rn(hidden.to(tl.float32), x_scale[:, None]),
[BLOCK_M, BLOCK_K // 16, 16],
)
block_scale = tl.max(tl.abs(values), axis=2) * (1.0 / 6.0)
block_scale = tl.minimum(tl.maximum(block_scale, 2.0**-9), 448.0)
block_scale = block_scale.to(tl.float8e4nv)
values = tl.div_rn(values, block_scale.to(tl.float32)[:, :, None])
magnitude = tl.abs(values)
# Round to nearest even on the grid {0, 0.5, 1, 1.5, 2, 3, 4, 6}.
code = (
(magnitude > 0.25).to(tl.int32)
+ (magnitude >= 0.75).to(tl.int32)
+ (magnitude > 1.25).to(tl.int32)
+ (magnitude >= 1.75).to(tl.int32)
+ (magnitude > 2.5).to(tl.int32)
+ (magnitude >= 3.5).to(tl.int32)
+ (magnitude > 5.0).to(tl.int32)
)
code = tl.where((values < 0) & (code != 0), code | 8, code)
low, high = tl.split(tl.reshape(code, [BLOCK_M, BLOCK_K // 2, 2]))
packed_offsets = k_block_id * (BLOCK_K // 2) + tl.arange(0, BLOCK_K // 2)
tl.store(
x_out + token_id[:, None] * x_stride_m + packed_offsets[None, :],
(low | (high << 4)).to(tl.uint8),
mask=token_mask[:, None],
)
sf_offsets = k_block_id * (BLOCK_K // 64) + tl.arange(0, BLOCK_K // 64)
sf_bytes = tl.reshape(
block_scale.to(tl.uint8, bitcast=True), [BLOCK_M, BLOCK_K // 64, 4]
)
tl.store(
x_sf + token_id[:, None] * x_sf_stride_m + sf_offsets[None, :],
_pack_bytes(sf_bytes, 4),
mask=token_mask[:, None],
)
if shared_x is not None:
shared_ptrs = shared_x + token_id[:, None] * shared_x_stride_m
if SHARED_MXFP8:
fp8, _, scale_exp = _quantize_fp8_groups(
hidden.to(tl.float32), BLOCK_M, BLOCK_K, 32, True
)
tl.store(shared_ptrs + k_ids[None, :], fp8, mask=token_mask[:, None])
tl.store(
shared_x_sf
+ _shared_sf_row(token_id, SHARED_BLOCK_M) * shared_x_sf_stride_m
+ k_block_id * shared_x_sf_stride_k,
_pack_bytes(scale_exp, BLOCK_K // 32),
mask=token_mask,
)
else:
tl.store(shared_ptrs + k_ids[None, :], hidden, mask=token_mask[:, None])
_store_topk(
token_id,
token_mask,
topk_ids,
topk_weights,
is_padding,
topk_idx_out,
topk_weights_out,
topk_ids_stride_m,
topk_ids_stride_k,
topk_weights_stride_m,
topk_weights_stride_k,
is_padding_stride_m,
topk_idx_stride_m,
topk_idx_stride_k,
topk_weights_out_stride_m,
topk_weights_out_stride_k,
top_k,
BLOCK_M,
BLOCK_TOPK,
)
def _resolve_hidden_quant(hidden_quant: QuantKey, block_k: int) -> tuple[int, bool]:
"""Map a hidden-state ``QuantKey`` to the kernel's ``(GROUP_K, USE_UE8M0)``."""
scale = hidden_quant.scale
group_shape = scale.group_shape
if (
hidden_quant.dtype != FP8_DTYPE
or not hidden_quant.symmetric
or hidden_quant.scale2 is not None
or scale.static
or not group_shape.is_per_group()
or block_k % group_shape.col != 0
or scale.dtype not in (*_E8M0_SCALE_DTYPES, torch.float32)
):
raise ValueError(
"MegaMoE input staging requires symmetric dynamic fp8 "
f"per-group quantization with a group size dividing {block_k} and "
f"E8M0 or fp32 scales, got {hidden_quant}."
)
return group_shape.col, scale.dtype in _E8M0_SCALE_DTYPES
def prepare_megamoe_inputs(
hidden_states: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
x: torch.Tensor,
x_sf: torch.Tensor | None,
topk_idx_out: torch.Tensor,
topk_weights_out: torch.Tensor,
is_padding: torch.Tensor | None = None,
shared_x_sf: torch.Tensor | None = None,
shared_block_m: int | None = None,
hidden_quant: QuantKey | None = kMxfp8Dynamic,
) -> None:
"""Quantize hidden states and repack top-k routing for DeepGEMM MegaMoE.
Args:
hidden_states: Input activations of shape ``[num_tokens, hidden_dim]``.
topk_weights: Router top-k weights of shape ``[num_tokens, top_k]``.
topk_ids: Router top-k expert ids of shape ``[num_tokens, top_k]``.
x: Output buffer for the dispatched hidden states.
x_sf: Output buffer for the hidden-state scale factors; ``None`` when
``hidden_quant`` is ``None``.
topk_idx_out: Output buffer for the repacked top-k expert ids.
topk_weights_out: Output buffer for the repacked top-k weights.
is_padding: Optional per-token mask; padded tokens are not routed.
shared_x_sf: Optional output buffer for shared-expert scale factors.
Must be given together with ``shared_block_m``.
shared_block_m: Block M used to lay out ``shared_x_sf``.
hidden_quant: Hidden-state quantization scheme. E8M0 scales are packed
four per int32 into ``x_sf``; fp32 scales are stored directly.
``None`` copies the hidden states into ``x`` unquantized.
"""
num_tokens, hidden_size = hidden_states.shape
if num_tokens == 0:
return
if hidden_size % 128 != 0:
raise ValueError(
"MegaMoE input staging requires hidden_size to be a multiple of 128."
)
block_k = 128
quantize = hidden_quant is not None
if quantize != (x_sf is not None):
raise ValueError("MegaMoE input staging needs x_sf iff hidden_quant is set.")
group_k, use_ue8m0 = (
(block_k, False)
if hidden_quant is None
else _resolve_hidden_quant(hidden_quant, block_k)
)
top_k = topk_ids.shape[1]
if topk_weights.shape != topk_ids.shape:
raise ValueError(
"MegaMoE input staging requires topk_weights and "
"topk_ids to have the same shape."
)
if (shared_x_sf is None) != (shared_block_m is None):
raise ValueError(
"MegaMoE shared input staging requires both shared_x_sf and shared_block_m."
)
if shared_x_sf is not None and not use_ue8m0:
raise ValueError(
"MegaMoE shared input staging currently requires "
"UE8M0-packed hidden scales."
)
if shared_x_sf is not None:
assert shared_block_m is not None
if shared_block_m <= 0:
raise ValueError("MegaMoE shared_block_m must be positive.")
expected_sf_k = hidden_size // 128
if shared_x_sf.ndim != 2 or shared_x_sf.shape[1] != expected_sf_k:
raise ValueError(
"MegaMoE shared_x_sf must have shape "
f"(*, {expected_sf_k}), got {tuple(shared_x_sf.shape)}."
)
aligned_block_m = triton.cdiv(shared_block_m, 128) * 128
required_rows = triton.cdiv(num_tokens, shared_block_m) * aligned_block_m
if shared_x_sf.shape[0] < required_rows:
raise ValueError(
"MegaMoE shared_x_sf has insufficient rows: requires "
f"{required_rows}, got {shared_x_sf.shape[0]}."
)
# On GB200, eight-row tiles win from 64 tokens; keep smaller batches untiled.
block_m = 8 if num_tokens >= 64 else 1
grid = (triton.cdiv(num_tokens, block_m), triton.cdiv(hidden_size, block_k))
block_topk = triton.next_power_of_2(top_k)
padding_stride_m = is_padding.stride(0) if is_padding is not None else 0
_prepare_megamoe_inputs_kernel[grid](
hidden_states,
x,
x_sf,
shared_x_sf,
topk_ids,
topk_weights,
is_padding,
topk_idx_out,
topk_weights_out,
hidden_states.stride(0),
hidden_states.stride(1),
x.stride(0),
x.stride(1),
x_sf.stride(0) if x_sf is not None else 0,
x_sf.stride(1) if x_sf is not None else 0,
shared_x_sf.stride(0) if shared_x_sf is not None else 0,
shared_x_sf.stride(1) if shared_x_sf is not None else 0,
topk_ids.stride(0),
topk_ids.stride(1),
topk_weights.stride(0),
topk_weights.stride(1),
padding_stride_m,
topk_idx_out.stride(0),
topk_idx_out.stride(1),
topk_weights_out.stride(0),
topk_weights_out.stride(1),
num_tokens,
hidden_size,
top_k,
BLOCK_M=block_m,
BLOCK_K=block_k,
GROUP_K=group_k,
BLOCK_TOPK=block_topk,
SHARED_BLOCK_M=shared_block_m or 1,
QUANTIZE=quantize,
USE_UE8M0=use_ue8m0,
num_warps=4,
)
def prepare_megamoe_nvfp4_inputs(
hidden_states: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
x: torch.Tensor,
x_sf: torch.Tensor,
x_scales: torch.Tensor,
topk_idx_out: torch.Tensor,
topk_weights_out: torch.Tensor,
is_padding: torch.Tensor | None = None,
shared_x: torch.Tensor | None = None,
shared_x_sf: torch.Tensor | None = None,
shared_block_m: int | None = None,
w2_global_scale: torch.Tensor | None = None,
l2_alpha: torch.Tensor | None = None,
l2_activation_scale: float = 1.0,
) -> None:
"""Stage DeepGEMM NVFP4 MegaMoE inputs (``kNvfp4DynamicToken``).
Each token is scaled by its own fp32 ``x_scales`` (row amax / 2688) and
quantized to E2M1 with one E4M3 scale per 16 values.
Args:
hidden_states: BF16 activations of shape ``[num_tokens, hidden_dim]``.
topk_weights: Router top-k weights of shape ``[num_tokens, top_k]``.
topk_ids: Router top-k expert ids of shape ``[num_tokens, top_k]``.
x: Packed E2M1 output, ``[num_tokens, hidden_dim // 2]``.
x_sf: E4M3 scales packed four per int32, ``[num_tokens, hidden_dim // 64]``.
x_scales: Per-token fp32 dequantization scales, ``[num_tokens]``.
topk_idx_out: Output buffer for the repacked top-k expert ids.
topk_weights_out: Output buffer for the repacked top-k weights.
is_padding: Optional per-token mask; padded tokens are not routed.
shared_x: Optional shared-expert input, fp8 (MXFP8 with ``shared_x_sf``)
or a BF16 copy of the hidden states.
shared_x_sf: UE8M0 scales of an fp8 ``shared_x`` in the shared-expert
TMA layout for ``shared_block_m``.
shared_block_m: Block M used to lay out ``shared_x_sf``.
w2_global_scale: Per-local-expert L2 weight global scales.
l2_alpha: Output ``w2_global_scale * l2_activation_scale``, the L2
dequantization scale of each local expert. Written even without
local tokens, as other ranks' tokens still reach these experts.
l2_activation_scale: Global scale the kernel quantizes L2 inputs with.
"""
num_tokens, hidden_size = hidden_states.shape
if hidden_size % 128 != 0:
raise ValueError(
"MegaMoE input staging requires hidden_size to be a multiple of 128."
)
if topk_weights.shape != topk_ids.shape:
raise ValueError(
"MegaMoE input staging requires topk_weights and "
"topk_ids to have the same shape."
)
shared_mxfp8 = shared_x is not None and shared_x.dtype == FP8_DTYPE
if shared_mxfp8 != (shared_x_sf is not None) or (shared_x_sf is None) != (
shared_block_m is None
):
raise ValueError(
"MegaMoE NVFP4 staging needs shared_x_sf and shared_block_m exactly "
"for an fp8 shared_x."
)
if (w2_global_scale is None) != (l2_alpha is None):
raise ValueError("MegaMoE NVFP4 staging needs w2_global_scale with l2_alpha.")
for name, tensor in (("x", x), ("x_sf", x_sf), ("shared_x", shared_x)):
if tensor is not None and tensor.stride(-1) != 1:
raise ValueError(f"MegaMoE NVFP4 staging requires a row-major {name}.")
block_m = 8 if num_tokens >= 64 else 1
top_k = topk_ids.shape[1]
num_local_experts = 0 if l2_alpha is None else l2_alpha.numel()
# One program even without tokens, to write l2_alpha.
grid = (max(triton.cdiv(num_tokens, block_m), 1),)
_prepare_megamoe_nvfp4_inputs_kernel[grid](
hidden_states,
x.view(torch.uint8),
x_sf,
x_scales,
shared_x,
shared_x_sf,
topk_ids,
topk_weights,
is_padding,
topk_idx_out,
topk_weights_out,
w2_global_scale,
l2_alpha,
hidden_states.stride(0),
hidden_states.stride(1),
x.stride(0),
x_sf.stride(0),
shared_x.stride(0) if shared_x is not None else 0,
shared_x_sf.stride(0) if shared_x_sf is not None else 0,
shared_x_sf.stride(1) if shared_x_sf is not None else 0,
topk_ids.stride(0),
topk_ids.stride(1),
topk_weights.stride(0),
topk_weights.stride(1),
is_padding.stride(0) if is_padding is not None else 0,
topk_idx_out.stride(0),
topk_idx_out.stride(1),
topk_weights_out.stride(0),
topk_weights_out.stride(1),
num_tokens,
l2_activation_scale,
hidden_size,
top_k,
num_local_experts,
BLOCK_M=block_m,
BLOCK_K=128,
BLOCK_TOPK=triton.next_power_of_2(top_k),
BLOCK_E=triton.next_power_of_2(max(num_local_experts, 1)),
SHARED_BLOCK_M=shared_block_m or 1,
SHARED_MXFP8=shared_mxfp8,
num_warps=4,
)
@@ -79,6 +79,9 @@ class RoutedExperts(PluggableLayer):
- Executing routed experts via quant_method.apply()
"""
# Hash-routing table, owned by the gate.
hash_indices_table: torch.Tensor | None = None
def __init__(
self,
layer_name: str,
@@ -107,6 +110,7 @@ class RoutedExperts(PluggableLayer):
swiglu_beta: float | None = None,
e_score_correction_bias: torch.Tensor | None = None,
apply_router_weight_on_input: bool = False,
hash_indices_table: torch.Tensor | None = None,
):
super().__init__()
self.layer_name = layer_name
@@ -152,6 +156,8 @@ class RoutedExperts(PluggableLayer):
# Set by bind_routed_experts_capturer for monolithic kernels.
self.routing_sink: RoutedExpertsSink | None = None
self.apply_router_weight_on_input = apply_router_weight_on_input
# The gate owns this Parameter: keep it out of this module's parameters.
object.__setattr__(self, "hash_indices_table", hash_indices_table)
# End random parameters
self._loaded_expert_biases: set[str] = set()
self._fused_shared_expert_quantizer = self._get_fused_shared_expert_quantizer()
@@ -14,7 +14,7 @@ import torch
from vllm.config import VllmConfig
from vllm.distributed.parallel_state import get_tp_group
from vllm.forward_context import get_forward_context
from vllm.forward_context import ForwardContext, get_forward_context
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
from vllm.platforms import current_platform
@@ -85,103 +85,74 @@ class RoutedExpertsCapturer:
self.tp_size = vllm_config.parallel_config.tensor_parallel_size
def capture(self, layer_id: int, topk_ids: torch.Tensor) -> None:
"""Capture expert routing decisions for a specific layer.
Under data parallelism, ``topk_ids`` may have four different batch
layouts depending on where the DP combine happens and whether
Expert Parallelism (EP) or Sequence Parallelism (SP) is active for the
MoE layer:
- ``n == total`` (naive dispatch): all DP ranks' tokens are
concatenated before routing; we slice out this rank's span
using the cumulative per-rank counts.
- ``n == token_num_per_dp`` (modular-kernel path): DP combine
happens inside ``quant_method.apply``; ``select_experts`` only
ever sees this rank's tokens, so we take the whole tensor.
- ``n == sum(dp_metadata.local_sizes)`` (naive DP+EP dispatch):
sequence-parallel shards from every DP rank are gathered through
the flattened EP group. The shard sizes include CUDA-graph / SP
padding, so we use them to locate this DP rank's unpadded rows.
- ``n == ceil(token_num_per_dp / tp_size)`` (SP + modular-kernel
path): tokens were split along dim=0 across the TP group by
``_sequence_parallel_context``
(``moe_runner_base.py:_sequence_parallel_context``), so each
TP rank only sees its shard. We all-gather along dim=0 to
reconstruct this DP rank's full routing tensor. SP pads with
ceil-div (see ``_compute_sp_num_tokens`` in
``forward_context.py``), so the gathered tensor may contain a
few trailing padding rows which are trimmed by the downstream
``[:token_num_per_dp]`` slice.
Args:
layer_id: The layer index.
topk_ids: Tensor of shape (batch_size, num_routed_experts).
"""
ctx = get_forward_context()
if ctx.dp_metadata is None:
local_topk_ids = topk_ids
else:
num_tokens_dp = ctx.dp_metadata.num_tokens_across_dp_cpu
num_local_tokens = int(num_tokens_dp[self.dp_rank].item())
total = int(num_tokens_dp.sum().item())
n = topk_ids.shape[0]
shard_sizes = getattr(ctx.dp_metadata, "local_sizes", None)
gathered_size = sum(shard_sizes) if shard_sizes is not None else -1
if n == total:
end = int(num_tokens_dp[: self.dp_rank + 1].sum().item())
local_topk_ids = topk_ids[end - num_local_tokens : end]
elif n == num_local_tokens:
local_topk_ids = topk_ids
elif shard_sizes is not None and n == gathered_size:
# Naive DP+EP dispatch gathers the sequence-parallel shards in
# flattened DP-then-TP order. ``local_sizes`` is the exact
# all-gatherv layout, including padding. Select this DP rank's
# contiguous shard group, then trim its trailing padding.
num_dp_ranks = len(num_tokens_dp)
assert len(shard_sizes) % num_dp_ranks == 0
shards_per_dp_rank = len(shard_sizes) // num_dp_ranks
first_shard = self.dp_rank * shards_per_dp_rank
start = sum(shard_sizes[:first_shard])
local_topk_ids = topk_ids[start : start + num_local_tokens]
elif (
self.tp_size > 1
and n == (num_local_tokens + self.tp_size - 1) // self.tp_size
):
# SP + modular-kernel path. All-gather across the TP
# group along dim=0 to reconstruct the full per-DP-rank
# tensor; keep only the first ``token_num_per_dp`` rows
# (trailing rows are SP ceil-div padding). The TP group
# is always initialized on real rollout workers, and
# every rank in the group reaches this branch in
# lockstep (bind is per-FusedMoEFactory layer, SP is a global
# condition), so a bare all_gather here will not
# deadlock -- let it raise if the precondition is
# violated rather than skip silently.
#
# ``topk_ids`` is already whatever the router produced
# (typically int32/int64, both supported by NCCL); the
# downstream buffer assignment narrows to the capture dtype.
local_topk_ids = get_tp_group().all_gather(topk_ids, dim=0)[
:num_local_tokens
]
else:
sp_expected = (num_local_tokens + self.tp_size - 1) // self.tp_size
raise AssertionError(
"RoutedExpertsCapturer: unexpected topk_ids batch "
f"dim {n} (expected {total}, {num_local_tokens}, "
f"{gathered_size}, or {sp_expected} for "
f"dp_rank={self.dp_rank}, tp_size={self.tp_size})"
)
if not 0 <= layer_id < self.device_buffer.shape[1]:
raise IndexError(
f"routed-experts layer {layer_id} exceeds capture buffer "
f"layer count {self.device_buffer.shape[1]}"
)
local_topk_ids = self._restore_local_topk_ids(topk_ids, get_forward_context())
self.device_buffer[: len(local_topk_ids), layer_id] = local_topk_ids
def _restore_local_topk_ids(
self, topk_ids: torch.Tensor, ctx: ForwardContext
) -> torch.Tensor:
"""Restore this DP rank's token rows, trimming trailing SP padding.
Layout priority: cross-DP batch, local batch, gathered DP/TP shards,
then a local TP shard. Gathered rows are DP-major/TP-minor; preserve
this priority when layout sizes coincide. PCP is unsupported.
Without DP metadata, use the pre-SP batch descriptor. Unknown counts
pass through; known counts must match a full, sharded or padded batch.
A full local batch avoids gathering, including zero/one-token batches
where only the publishing TP0 rank needs the real rows.
"""
n = topk_ids.shape[0]
dp_metadata = ctx.dp_metadata
if dp_metadata is None:
if ctx.batch_descriptor is None:
return topk_ids
num_local_tokens = ctx.batch_descriptor.num_tokens
else:
num_tokens_dp = dp_metadata.num_tokens_across_dp_cpu
num_local_tokens = int(num_tokens_dp[self.dp_rank].item())
if n == int(num_tokens_dp.sum().item()):
end = int(num_tokens_dp[: self.dp_rank + 1].sum().item())
return topk_ids[end - num_local_tokens : end]
if n == num_local_tokens:
return topk_ids
if dp_metadata is not None:
shard_sizes = dp_metadata.local_sizes
if shard_sizes is not None and n == sum(shard_sizes):
num_dp_ranks = len(num_tokens_dp)
if len(shard_sizes) % num_dp_ranks != 0:
raise ValueError(
"RoutedExpertsCapturer: gathered shard count "
f"{len(shard_sizes)} is not divisible by DP rank count "
f"{num_dp_ranks}"
)
shards_per_dp_rank = len(shard_sizes) // num_dp_ranks
first_shard = self.dp_rank * shards_per_dp_rank
start = sum(shard_sizes[:first_shard])
return topk_ids[start : start + num_local_tokens]
shard_n = (num_local_tokens + self.tp_size - 1) // self.tp_size
if n == shard_n:
return get_tp_group().all_gather(topk_ids.contiguous(), dim=0)[
:num_local_tokens
]
if dp_metadata is None and n == shard_n * self.tp_size:
return topk_ids[:num_local_tokens]
raise AssertionError(
f"RoutedExpertsCapturer: unexpected topk_ids batch dim {n} "
f"(local_tokens={num_local_tokens}, "
f"dp_rank={self.dp_rank}, tp_size={self.tp_size})"
)
def snapshot_routing_data(self, num_tokens: int) -> torch.Tensor:
"""Return a stable snapshot of the current routing data."""
return self.device_buffer[:num_tokens].to(self.output_dtype)
@@ -84,7 +84,14 @@ class AiterSharedRoutedFusedMoERouter(BaseRouter):
total_topk_weights_slice = total_topk_weights[:M]
topk_ids_slice = total_topk_ids[:M, :topk]
topk_func = dispatch_topk_softmax_func(use_rocm_aiter=True)
topk_func = dispatch_topk_softmax_func(
use_rocm_aiter=True,
topk_weights=total_topk_weights_slice,
topk_indices=topk_ids_slice,
gating_output=router_logits,
num_shared_experts=num_fse,
shared_expert_scoring_func="sigmoid",
)
topk_func(
total_topk_weights_slice,
topk_ids_slice,
@@ -110,7 +117,10 @@ class AiterSharedRoutedFusedMoERouter(BaseRouter):
)
topk_func = dispatch_topk_softmax_func(
use_rocm_aiter=rocm_aiter_ops.is_fused_moe_enabled()
use_rocm_aiter=rocm_aiter_ops.is_fused_moe_enabled(),
topk_weights=topk_weights,
topk_indices=topk_ids,
gating_output=routing_logits,
)
topk_weights, topk_ids = topk_func(
topk_weights,
@@ -61,10 +61,73 @@ def vllm_topk_sigmoid(
return topk_weights, topk_indices
# At E=512 / top-10, AITER's topk_gating is faster than the legacy launcher
# through this many tokens; its one-row generic prefill path regresses above it.
_AITER_TOPK_GATING_MAX_TOKENS = 4096
def _aiter_topk_gating_supported(
topk_weights: torch.Tensor,
topk_indices: torch.Tensor,
gating_output: torch.Tensor,
num_shared_experts: int,
shared_expert_scoring_func: str,
) -> bool:
"""Whether AITER's ``topk_gating`` can serve this softmax top-k launch.
It needs contiguous gating rows and at most ``_AITER_TOPK_GATING_MAX_TOKENS``
tokens. Fused shared experts are scored with sigmoid after the routed
top-k: 1, 2, 4 or 8 of them, with ``topk_weights`` and ``topk_indices``
sharing a row stride and ``topk_weights`` wide enough for the shared
columns. Shared expert ids stay as the caller prefilled them.
"""
if (
not gating_output.is_contiguous()
or gating_output.shape[0] > _AITER_TOPK_GATING_MAX_TOKENS
):
return False
if num_shared_experts == 0:
return not shared_expert_scoring_func
return (
shared_expert_scoring_func == "sigmoid"
and num_shared_experts in (1, 2, 4, 8)
and topk_weights.stride(0) == topk_indices.stride(0)
and topk_weights.shape[-1] >= topk_indices.shape[-1] + num_shared_experts
)
def dispatch_topk_softmax_func(
use_rocm_aiter: bool = False,
*,
topk_weights: torch.Tensor | None = None,
topk_indices: torch.Tensor | None = None,
gating_output: torch.Tensor | None = None,
num_shared_experts: int = 0,
shared_expert_scoring_func: str = "",
) -> Callable[..., tuple[torch.Tensor, ...]]:
"""Pick the softmax top-k implementation for one launch.
On ROCm with AITER, ``topk_gating`` is chosen on gfx942 and gfx950 when the
launch described by the tensors and shared-expert arguments is one it supports;
every other AITER launch uses the legacy ``topk_softmax``. The tensors are
optional: without them the launch cannot be judged and the legacy launcher
is used.
"""
if use_rocm_aiter:
if (
rocm_aiter_ops.is_topk_gating_enabled()
and topk_weights is not None
and topk_indices is not None
and gating_output is not None
and _aiter_topk_gating_supported(
topk_weights,
topk_indices,
gating_output,
num_shared_experts,
shared_expert_scoring_func,
)
):
return rocm_aiter_ops.topk_gating
return rocm_aiter_ops.topk_softmax
return vllm_topk_softmax
@@ -104,7 +167,10 @@ def fused_topk(
if scoring_func == "softmax":
topk_func = dispatch_topk_softmax_func(
use_rocm_aiter=rocm_aiter_ops.is_fused_moe_enabled()
use_rocm_aiter=rocm_aiter_ops.is_fused_moe_enabled(),
topk_weights=topk_weights,
topk_indices=topk_ids,
gating_output=gating_output,
)
topk_weights, topk_ids = topk_func(
topk_weights, topk_ids, token_expert_indices, gating_output, renormalize
@@ -24,14 +24,15 @@ class GateLinear(ReplicatedLinear):
1. cuteDSL ll_bf16_gemm (SM90+, M<=16, bf16 in, fp32 out,
any shape w/ K divisible by 8)
2. fp32 specialized kernel (SM90+ or gfx950, bf16/fp32 in, fp32 out,
M<=32, model-specific shapes)
2. fp32 specialized kernel (bf16/fp32 in, fp32 out, M<=32; SM90+
model-specific shapes, gfx950 any shape)
3. cuteDSL ll_fp32w_gemm (SM90+, M<=32, bf16/fp32 in, fp32 out,
any shape)
4. bf16x3 CuteDSL kernel (SM100, bf16 in, fp32 weight)
5. ROCm bf16x3 router GEMM (gfx950, bf16 in, fp32 weight, fp32 out,
M>=2048)
6. cuBLAS bf16×bf16→fp32 (SM90+ + bf16 weight + fp32 out_dtype)
6. cuBLAS bf16×bf16→fp32 (SM90+ + bf16 weight + fp32 out_dtype); gfx950
routes low M through the tier 2 kernel
7. F.linear via ReplicatedLinear (ultimate fallback)
The ``out_dtype`` attribute is mutable and can be set after init
@@ -73,16 +74,6 @@ class GateLinear(ReplicatedLinear):
from vllm.platforms.rocm import on_gfx950
is_gfx950 = on_gfx950()
is_rocm_fp32_shape = False
if is_gfx950:
from vllm.model_executor.layers.fused_moe.router.rocm_fp32_router_gemm import ( # noqa: E501
ROCM_FP32_ROUTER_GEMM_SUPPORTED_SHAPES,
)
is_rocm_fp32_shape = (
input_size,
output_size,
) in ROCM_FP32_ROUTER_GEMM_SUPPORTED_SHAPES
batch_invariant = envs.VLLM_BATCH_INVARIANT
can_use_specialized_kernels = (
current_platform.is_cuda()
@@ -107,6 +98,7 @@ class GateLinear(ReplicatedLinear):
return_bias=return_bias,
)
self.out_dtype = out_dtype
self.is_gfx950 = is_gfx950
# A quantized gate exposes no plain ``weight``, so every specialized
# tier below is disabled and forward falls back to ReplicatedLinear.
@@ -127,7 +119,7 @@ class GateLinear(ReplicatedLinear):
and (is_hopper or is_blackwell)
and (input_size, output_size) in self.FP32_SUPPORTED_SHAPES
)
or (is_gfx950 and is_rocm_fp32_shape)
or is_gfx950
)
)
self.allow_bf16x3_router_gemm = (
@@ -349,7 +341,10 @@ class GateLinear(ReplicatedLinear):
# Tier 6: cuBLAS bf16→fp32
if self.allow_cublas_router_gemm and x.dtype == torch.bfloat16:
output = torch.mm(x, self.weight.T, out_dtype=torch.float32)
if self.is_gfx950:
output = torch.ops.vllm.fp32_router_gemm_dispatch(x, self.weight, False)
else:
output = torch.mm(x, self.weight.T, out_dtype=torch.float32)
return self._return(output)
# Tier 7: F.linear (ReplicatedLinear)
@@ -465,17 +460,20 @@ def fp32_router_gemm_dispatch_impl(
This must be wrapped in a custom op because our torch.compile integration
does not support runtime dispatching on num_tokens.
"""
if x.shape[0] <= _FP32_ROUTER_GEMM_MAX_TOKENS:
if current_platform.is_rocm():
from vllm.model_executor.layers.fused_moe.router.rocm_fp32_router_gemm import ( # noqa: E501
can_use_rocm_fp32_router_gemm,
rocm_fp32_router_gemm,
)
if current_platform.is_rocm():
from vllm.model_executor.layers.fused_moe.router.rocm_fp32_router_gemm import ( # noqa: E501
can_use_rocm_fp32_router_gemm,
rocm_fp32_router_gemm,
)
if x.shape[0] <= _FP32_ROUTER_GEMM_MAX_TOKENS:
x = x.contiguous()
if can_use_rocm_fp32_router_gemm(x, weight):
return rocm_fp32_router_gemm(x, weight)
return torch.nn.functional.linear(x.float(), weight)
if weight.dtype == torch.bfloat16:
return torch.mm(x, weight.T, out_dtype=torch.float32)
return torch.nn.functional.linear(x.float(), weight)
if x.shape[0] <= _FP32_ROUTER_GEMM_MAX_TOKENS:
return ops.fp32_router_gemm(x, weight)
if allow_bf16x3_router_gemm and x.dtype == torch.bfloat16:
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Low-token FP32 router GEMM for gfx950."""
"""Low-token router GEMM with FP32 output for gfx950 (FP32 or BF16 weights)."""
import torch
@@ -9,15 +9,10 @@ from vllm.platforms.rocm import on_gfx950
from vllm.triton_utils import tl, triton
_MAX_TOKENS = 32
ROCM_FP32_ROUTER_GEMM_SUPPORTED_SHAPES = frozenset(
{
(3072, 256),
(4096, 8),
(4096, 192),
(6144, 128),
(6144, 256),
}
)
# From this many multiply-adds on, F.linear wins on MI355.
_MAX_MACS = 1 << 27
# Above this num_tokens * num_experts, hipBLASLt wins for BF16 weights.
_MAX_BF16_TOKEN_EXPERTS = 8192
@triton.jit
@@ -30,6 +25,7 @@ def _rocm_fp32_router_gemm_kernel(
N: tl.constexpr,
BLOCK_M: tl.constexpr,
BLOCK_K: tl.constexpr,
EVEN_K: tl.constexpr,
):
pid = tl.program_id(0)
pid_m = pid // N
@@ -40,12 +36,27 @@ def _rocm_fp32_router_gemm_kernel(
for k_start in range(0, K, BLOCK_K):
offsets_k_block = k_start + offsets_k
hidden_states = tl.load(
hidden_states_ptr + offsets_m[:, None] * K + offsets_k_block[None, :],
mask=offsets_m[:, None] < M,
other=0.0,
).to(tl.float32)
router_weight = tl.load(router_weight_ptr + pid_n * K + offsets_k_block)
if EVEN_K:
hidden_states = tl.load(
hidden_states_ptr + offsets_m[:, None] * K + offsets_k_block[None, :],
mask=offsets_m[:, None] < M,
other=0.0,
).to(tl.float32)
router_weight = tl.load(router_weight_ptr + pid_n * K + offsets_k_block).to(
tl.float32
)
else:
k_mask = offsets_k_block < K
hidden_states = tl.load(
hidden_states_ptr + offsets_m[:, None] * K + offsets_k_block[None, :],
mask=(offsets_m[:, None] < M) & k_mask[None, :],
other=0.0,
).to(tl.float32)
router_weight = tl.load(
router_weight_ptr + pid_n * K + offsets_k_block,
mask=k_mask,
other=0.0,
).to(tl.float32)
partials += hidden_states * router_weight[None, :]
accumulator = tl.sum(partials, axis=1)
@@ -57,28 +68,36 @@ def _rocm_fp32_router_gemm_kernel(
def _launch_config(
hidden_size: int,
num_experts: int,
num_tokens: int,
num_experts: int, num_tokens: int, bf16_weight: bool
) -> tuple[int, int, int]:
if (hidden_size, num_experts) == (4096, 8):
return 1, 1024, 4
"""Return (BLOCK_M, BLOCK_K, num_warps), swept on MI355 for M<=32.
One program computes BLOCK_M tokens for one expert, so narrow routers keep
BLOCK_M=1 to fill the GPU and wide routers grow BLOCK_M to reuse weights.
"""
if bf16_weight:
return (1 if num_tokens <= 4 else 2), 1024, 2
if num_experts < 64:
return 1, 1024, 4
wide = num_experts > 256
if num_tokens <= 2:
return 1, 512, 2
if num_tokens <= 4:
return 1, 1024, 8
return (2, 1024, 2) if wide else (2, 2048, 4)
if num_tokens <= 8:
return (4, 512, 2) if wide else (4, 1024, 4)
if num_tokens <= 16:
block_m = 8 if (hidden_size, num_experts) == (6144, 256) else 4
return block_m, 1024, 8
if (hidden_size, num_experts) == (3072, 256):
return 8, 1024, 8
return 4, 2048, 8
return (8, 512, 2) if wide else (4, 1024, 4)
if num_experts >= 768:
return 16, 256, 2
return (8, 256, 2) if wide else (4, 512, 2)
def can_use_rocm_fp32_router_gemm(
hidden_states: torch.Tensor,
router_weight: torch.Tensor,
) -> bool:
"""Return whether the tensors match the tuned gfx950 fast path."""
"""Return whether the tensors can use the gfx950 fast path."""
try:
_validate_inputs(hidden_states, router_weight)
except (RuntimeError, ValueError):
@@ -96,26 +115,30 @@ def _validate_inputs(
raise ValueError("hidden_states and router_weight must be 2D tensors")
if hidden_states.dtype not in (torch.bfloat16, torch.float32):
raise ValueError("hidden_states must have dtype bfloat16 or float32")
if router_weight.dtype != torch.float32:
raise ValueError("router_weight must have dtype float32")
if router_weight.dtype not in (torch.float32, torch.bfloat16):
raise ValueError("router_weight must have dtype float32 or bfloat16")
if hidden_states.device.type != "cuda" or router_weight.device.type != "cuda":
raise ValueError("hidden_states and router_weight must be GPU tensors")
if hidden_states.device != router_weight.device:
raise ValueError("hidden_states and router_weight must be on the same device")
if not hidden_states.is_contiguous() or not router_weight.is_contiguous():
raise ValueError("hidden_states and router_weight must be contiguous")
shape = (hidden_states.shape[1], router_weight.shape[0])
if (
shape not in ROCM_FP32_ROUTER_GEMM_SUPPORTED_SHAPES
or router_weight.shape[1] != shape[0]
):
raise ValueError(
"supported (hidden_size, num_experts) shape pairs are "
"(3072, 256), (4096, 8), (4096, 192), (6144, 128), "
"and (6144, 256)"
)
if router_weight.shape[1] != hidden_states.shape[1]:
raise ValueError("router_weight and hidden_states hidden sizes differ")
if not 0 <= hidden_states.shape[0] <= _MAX_TOKENS:
raise ValueError(f"num_tokens must be in [0, {_MAX_TOKENS}]")
if hidden_states.shape[0] * router_weight.numel() >= _MAX_MACS:
raise ValueError(
f"num_tokens * num_experts * hidden_size must be < {_MAX_MACS}"
)
if (
router_weight.dtype == torch.bfloat16
and hidden_states.shape[0] * router_weight.shape[0] > _MAX_BF16_TOKEN_EXPERTS
):
raise ValueError(
f"num_tokens * num_experts must be <= {_MAX_BF16_TOKEN_EXPERTS} "
"for bfloat16 weights"
)
def rocm_fp32_router_gemm(
@@ -131,7 +154,10 @@ def rocm_fp32_router_gemm(
if num_tokens == 0:
return output
block_m, block_k, num_warps = _launch_config(hidden_size, num_experts, num_tokens)
block_m, block_k, num_warps = _launch_config(
num_experts, num_tokens, router_weight.dtype == torch.bfloat16
)
block_k = min(block_k, triton.next_power_of_2(hidden_size))
grid = (triton.cdiv(num_tokens, block_m) * num_experts,)
_rocm_fp32_router_gemm_kernel[grid](
hidden_states,
@@ -142,6 +168,7 @@ def rocm_fp32_router_gemm(
N=num_experts,
BLOCK_M=block_m,
BLOCK_K=block_k,
EVEN_K=hidden_size % block_k == 0,
num_warps=num_warps,
num_stages=1,
)
@@ -325,6 +325,7 @@ def _fp8_quantize(
A_scale: torch.Tensor | None,
per_act_token: bool,
block_shape: list[int] | None = None,
is_scale_transposed: bool = False,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Perform fp8 quantization on the inputs. If a block_shape
is provided, the output will be blocked.
@@ -339,7 +340,9 @@ def _fp8_quantize(
assert not per_act_token
assert len(block_shape) == 2
_, block_k = block_shape[0], block_shape[1]
A, A_scale = per_token_group_quant_fp8(A, block_k)
A, A_scale = per_token_group_quant_fp8(
A, block_k, column_major_scales=is_scale_transposed
)
assert cdiv(A.size(-1), block_k) == A_scale.size(-1)
return A, A_scale
@@ -462,6 +465,7 @@ def moe_kernel_quantize_input(
per_act_token_quant: bool,
block_shape: list[int] | None = None,
is_scale_swizzled: bool = True,
is_scale_transposed: bool = False,
quantization_emulation: bool = False,
mx_alignment: int = 0,
) -> tuple[torch.Tensor, torch.Tensor | None]:
@@ -469,7 +473,9 @@ def moe_kernel_quantize_input(
if quantization_emulation:
return _fp8_quantize_dequantize(A, A_scale)
else:
return _fp8_quantize(A, A_scale, per_act_token_quant, block_shape)
return _fp8_quantize(
A, A_scale, per_act_token_quant, block_shape, is_scale_transposed
)
elif quant_dtype == torch.int8:
if quantization_emulation:
raise NotImplementedError(
+3
View File
@@ -31,6 +31,7 @@ from vllm.model_executor.layers.quantization.base_config import (
)
from vllm.model_executor.layers.utils import (
dispatch_unquantized_gemm,
maybe_pad_weight_avoid_cache_aliasing,
)
from vllm.model_executor.parameter import (
BasevLLMParameter,
@@ -225,6 +226,8 @@ class UnquantizedLinearMethod(LinearMethodBase):
and weight.stride(0) != 1
):
layer.weight.data = weight.t().contiguous().t()
elif current_platform.is_rocm():
layer.weight.data = maybe_pad_weight_avoid_cache_aliasing(layer.weight.data)
def apply(
self,
@@ -479,6 +479,8 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
input_ids=input_ids,
hash_indices_table=layer.hash_indices_table,
)
@@ -1132,4 +1134,6 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
input_ids=input_ids,
hash_indices_table=layer.hash_indices_table,
)
@@ -0,0 +1,204 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Online-quantized copies of an already-loaded lm_head."""
from itertools import chain
from typing import Literal
from weakref import WeakKeyDictionary
import torch
from torch.nn import Module, Parameter
from vllm.logger import init_logger
from vllm.model_executor.kernels.linear import init_nvfp4_linear_kernel
from vllm.model_executor.layers.linear import UnquantizedLinearMethod
from vllm.model_executor.layers.quantization.online.fp8 import (
Fp8PtpcOnlineLinearMethod,
OnlineLinearBase,
)
from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import (
FLOAT4_E2M1_MAX,
ref_nvfp4_quant,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import weight_amax
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
UnquantizedEmbeddingMethod,
)
from vllm.model_executor.model_loader.reload.layerwise import get_layerwise_info
from vllm.model_executor.utils import replace_parameter
from vllm.utils.torch_utils import set_default_torch_dtype
logger = init_logger(__name__)
def _quantize_nvfp4(
weight: torch.Tensor, chunk_rows: int = 8192
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""NVFP4-quantize the rows of `weight` without the SM100+ quant kernel.
Returns packed E2M1 codes (low nibble first), E4M3 scales per 16 values and
the fp32 dequantization global scale, in the checkpoint layout.
"""
fp8_max = torch.finfo(torch.float8_e4m3fn).max
global_scale = (weight_amax(weight).float() / (FLOAT4_E2M1_MAX * fp8_max)).clamp(
min=1e-12
)
inv_global_scale = (1.0 / global_scale).reshape(())
# E2M1 magnitudes 0, 0.5, ..., 6 doubled to integers, mapped to their codes.
code_of = torch.zeros(13, dtype=torch.uint8, device=weight.device)
code_of[torch.tensor([0, 1, 2, 3, 4, 6, 8, 12])] = torch.arange(
8, dtype=torch.uint8
).to(weight.device)
n, k = weight.shape
packed = torch.empty(n, k // 2, dtype=torch.uint8, device=weight.device)
scales = torch.empty(n, k // 16, dtype=torch.float8_e4m3fn, device=weight.device)
for start in range(0, n, chunk_rows):
rows = slice(start, start + chunk_rows)
values, block_scales = ref_nvfp4_quant(weight[rows], inv_global_scale, 16)
codes = code_of[(values.abs() * 2).long()] | ((values < 0).to(torch.uint8) << 3)
packed[rows] = codes[:, 0::2] | (codes[:, 1::2] << 4)
scales[rows] = block_scales.to(torch.float8_e4m3fn)
return packed, scales, global_scale
class _LMHeadNvfp4Method(OnlineLinearBase):
"""Weight-only NVFP4 lm_head copy on the platform's W4A16 kernel;
activations stay in the model dtype."""
def process_weights_after_loading(self, layer: Module) -> None:
packed, scales, global_scale = _quantize_nvfp4(layer.weight)
replace_parameter(layer, "weight", packed)
layer.weight_scale = Parameter(scales, requires_grad=False)
layer.weight_global_scale = Parameter(global_scale, requires_grad=False)
self.kernel = init_nvfp4_linear_kernel(use_a16=True)
self.kernel.process_weights_after_loading(layer)
def apply(
self, layer: Module, x: torch.Tensor, bias: torch.Tensor | None = None
) -> torch.Tensor:
return self.kernel.apply_weights(layer, x, bias)
_LM_HEAD_METHODS: dict[str, type[OnlineLinearBase]] = {
"fp8": Fp8PtpcOnlineLinearMethod,
"nvfp4": _LMHeadNvfp4Method,
}
def _quantize_copy(lm_head: ParallelLMHead, quantization: str) -> ParallelLMHead:
if (
not isinstance(
lm_head.quant_method, (UnquantizedEmbeddingMethod, UnquantizedLinearMethod)
)
or getattr(lm_head, "bias", None) is not None
or lm_head.weight.dtype not in (torch.float16, torch.bfloat16)
):
raise ValueError(
"Quantizing an lm_head copy needs an unquantized fp16/bf16 lm_head "
"without bias."
)
# Online methods take their output dtype from the default dtype, which is
# only the model dtype while the model itself is being built.
dtype = lm_head.weight.dtype
with set_default_torch_dtype(dtype):
head = ParallelLMHead(
lm_head.num_embeddings,
lm_head.embedding_dim,
params_dtype=dtype,
org_num_embeddings=lm_head.org_vocab_size,
padding_size=lm_head.padding_size,
prefix=lm_head.prefix,
disable_tp=lm_head.disable_tp,
quant_method=_LM_HEAD_METHODS[quantization](),
)
replace_parameter(head, "weight", lm_head.weight.data)
head.quant_method.process_weights_after_loading(head)
# The copy is never loaded. The online loader wrapper captures the layer,
# so keeping it would leave each copy to the cycle collector.
for param in head.parameters(recurse=False):
if hasattr(param, "weight_loader"):
del param.weight_loader
return head
# Each quantized copy maps to the head it is derived from. Weak keys, and no
# module attribute, so the copy never owns the source as a submodule.
_SOURCES: WeakKeyDictionary[Module, tuple[ParallelLMHead, str]] = WeakKeyDictionary()
def quantized_lm_head_copy(
lm_head: ParallelLMHead, quantization: Literal["fp8", "nvfp4"]
) -> ParallelLMHead:
"""A ParallelLMHead holding this rank's shard of `lm_head`, quantized.
"fp8" uses per-row weight and per-token activation scales on the platform's
FP8 linear kernel; "nvfp4" is weight-only on the
platform's W4A16 kernel. `lm_head` is left
unchanged and no bf16 copy of its weight is made. The copy is derived state:
`refresh_quantized_lm_heads` re-derives it after `lm_head` changes.
"""
head = _quantize_copy(lm_head, quantization)
# Keep it out of layerwise reload, which would re-run
# process_weights_after_loading on the already-quantized weight.
get_layerwise_info(head).reset()
_SOURCES[head] = (lm_head, quantization)
logger.info(
"Quantized a %s copy of the lm_head (%d x %d per rank).",
quantization,
head.output_size_per_partition,
head.input_size_per_partition,
)
return head
@torch.no_grad()
def refresh_quantized_lm_heads(model: Module) -> None:
"""Re-derive the quantized lm_head copies in `model` from their source heads.
Copies in place, so CUDA graphs keep pointing at valid tensors.
"""
for module in model.modules():
if module not in _SOURCES:
continue
fresh = _quantize_copy(*_SOURCES[module])
for name, tensor in chain(
fresh.named_parameters(recurse=False), fresh.named_buffers(recurse=False)
):
getattr(module, name).copy_(tensor)
# (parent, attribute, copy) for each copy swapped out by share_source_lm_heads.
_SWAPPED: dict[Module, list[tuple[Module, str, Module]]] = {}
def share_source_lm_heads(model: Module) -> None:
"""Replace each quantized copy in `model` with its source head.
`model` then loads weights like a drafter sharing the unquantized head,
until `restore_quantized_lm_heads` swaps the copies back in.
"""
swapped = [
(module, name, child)
for module in model.modules()
for name, child in module.named_children()
if child in _SOURCES
]
for module, name, child in swapped:
setattr(module, name, _SOURCES[child][0])
_SWAPPED[model] = swapped
def restore_quantized_lm_heads(model: Module, refresh: bool = True) -> None:
"""Undo `share_source_lm_heads`, then re-derive the copies in `model`.
Args:
model: The model passed to `share_source_lm_heads`, if any.
refresh: Whether to re-derive the copies; False when the source heads
may be mid-reload.
"""
for module, name, head in _SWAPPED.pop(model, ()):
setattr(module, name, head)
if refresh:
refresh_quantized_lm_heads(model)
@@ -1647,8 +1647,10 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
# If no native backend available, use emulation.
if self.mxfp4_backend is Mxfp4MoeBackend.NONE:
self.mxfp4_backend = Mxfp4MoeBackend.EMULATION
self.experts_cls = None
self.experts_cls = backend_to_kernel_cls(self.mxfp4_backend)[0]
if self.experts_cls is None:
self.experts_cls = backend_to_kernel_cls(self.mxfp4_backend)[0]
logger.info_once(
f"Using {self.mxfp4_backend.value} backend for {self.ocp_mx_scheme}"
@@ -379,6 +379,79 @@ def align_moe_weights_for_fi(
return padded_w13, padded_w2, padded_intermediate
def align_mxfp8_moe_weights_for_fi_trtllm(
w13: torch.Tensor,
w2: torch.Tensor,
w13_scale: torch.Tensor,
w2_scale: torch.Tensor,
is_act_and_mul: bool,
min_alignment: int,
hidden_alignment: int,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, int, int]:
"""align_moe_weights_for_fi for MXFP8 weights and their UE8M0 block scales
for the FlashInfer TRT-LLM MoE kernel, which also pads the hidden size to a
multiple of hidden_alignment.
Padded weights are zero and padded scales are 127 (UE8M0 for 1.0), so the
padded rows and columns contribute nothing. Scales may be 3D
[E, rows, cols] or flattened [E, rows * cols]; their layout is kept.
Returns the padded intermediate and hidden sizes.
"""
num_experts, hidden_size, intermediate = w2.shape
padded_hidden = round_up(hidden_size, hidden_alignment)
w13, w2, padded_intermediate = align_moe_weights_for_fi(
w13, w2, is_act_and_mul, min_alignment
)
if padded_intermediate == intermediate and padded_hidden == hidden_size:
return w13, w2, w13_scale, w2_scale, intermediate, hidden_size
if padded_hidden != hidden_size:
logger.info_once(
"Padding hidden size from %d to %d for MXFP8 TRTLLM MoE kernel.",
hidden_size,
padded_hidden,
)
padded_w13 = w13.new_zeros((num_experts, w13.shape[1], padded_hidden))
padded_w13[:, :, :hidden_size] = w13
padded_w2 = w2.new_zeros((num_experts, padded_hidden, w2.shape[2]))
padded_w2[:, :hidden_size] = w2
w13, w2 = padded_w13, padded_w2
up_mult = 2 if is_act_and_mul else 1
w13_scale_3d = w13_scale.reshape(num_experts, up_mult * intermediate, -1)
w2_scale_3d = w2_scale.reshape(num_experts, hidden_size, -1)
block = hidden_size // w13_scale_3d.shape[2]
assert padded_hidden % block == 0 and padded_intermediate % block == 0
# Pad each of the gate and up halves (or the single non-gated block) on its
# own, like the weights.
padded_w13_scale = w13_scale_3d.new_full(
(num_experts, up_mult * padded_intermediate, padded_hidden // block), 127
)
for i in range(up_mult):
dst = i * padded_intermediate
padded_w13_scale[:, dst : dst + intermediate, : w13_scale_3d.shape[2]] = (
w13_scale_3d[:, i * intermediate : (i + 1) * intermediate]
)
padded_w2_scale = w2_scale_3d.new_full(
(num_experts, padded_hidden, padded_intermediate // block), 127
)
padded_w2_scale[:, :hidden_size, : w2_scale_3d.shape[2]] = w2_scale_3d
if w13_scale.ndim == 2:
padded_w13_scale = padded_w13_scale.reshape(num_experts, -1)
if w2_scale.ndim == 2:
padded_w2_scale = padded_w2_scale.reshape(num_experts, -1)
return (
w13,
w2,
padded_w13_scale,
padded_w2_scale,
padded_intermediate,
padded_hidden,
)
def _shuffle_deepseek_fp8_moe_weights(
w13: torch.Tensor,
w2: torch.Tensor,
@@ -486,8 +559,9 @@ def prepare_fp8_moe_layer_for_fi(
Note that for trtllm we update the model state dict
with the scale format needed for these kernels.
Note that for per-tensor, we update the layer's
intermediate size if the weights needed padding.
Note that for per-tensor and for MXFP8 TRT-LLM, we update the layer's
intermediate size (and for MXFP8 TRT-LLM its hidden size) if the weights
needed padding.
"""
assert hasattr(layer.moe_config, "is_act_and_mul")
block_quant = (
@@ -499,6 +573,21 @@ def prepare_fp8_moe_layer_for_fi(
# MXFP8 TRT-LLM requires W31 swap + reorder + shuffle.
if is_mxfp8 and is_trtllm:
# The kernel needs intermediate % 128 == 0 and hidden % 256 == 0. The
# MoE runner pads the activations to the padded hidden size.
w13, w2, w13_scale, w2_scale, new_intermediate, new_hidden = (
align_mxfp8_moe_weights_for_fi_trtllm(
w13,
w2,
w13_scale,
w2_scale,
layer.moe_config.is_act_and_mul,
min_alignment=128,
hidden_alignment=256,
)
)
layer.moe_config.intermediate_size_per_partition = new_intermediate
layer.moe_config.hidden_dim = new_hidden
# FlashInfer TRT-LLM SwiGLU expects [up; gate] but vLLM stores
# [gate; up]. Swap both weights and scales before interleaving.
if layer.moe_config.is_act_and_mul:
@@ -13,6 +13,14 @@ from vllm.utils.torch_utils import direct_register_custom_op, is_torch_equal_or_
logger = init_logger(__name__)
def should_use_gfx1250_mx_scale_swizzle() -> bool:
if not current_platform.is_rocm():
return False
from vllm.platforms.rocm import on_gfx1250
return on_gfx1250()
def should_use_cdna4_mx_scale_swizzle() -> bool:
"""Whether to use the CDNA4 swizzled scale layout for mxfp4 on gfx950.
@@ -27,6 +35,15 @@ def should_use_cdna4_mx_scale_swizzle() -> bool:
return on_gfx950() and get_tensor_model_parallel_world_size() <= 2
def get_aiter_mx_scale_swizzle() -> str | None:
"""AITER `swizzle_mx_scale` arg matching the load-time scale swizzle."""
if should_use_gfx1250_mx_scale_swizzle():
return "GFX1250_SCALE"
if should_use_cdna4_mx_scale_swizzle():
return "CDNA4_SCALE"
return None
def weight_mx_scale(precision_config):
"""mxfp4 weight scale from a triton_kernels PrecisionConfig: 3.8 exposes it
as b_mx_scale, 3.5.1/3.6 as weight_scale.
@@ -136,6 +153,17 @@ def _swizzle_mxfp4(quant_tensor, scale, num_warps=8):
# transpose the tensor so that the quantization axis is on dim1
quant_tensor = quant_tensor.transpose(-2, -1)
scale = scale.transpose(-2, -1)
if should_use_gfx1250_mx_scale_swizzle():
from aiter.ops.triton.utils.shuffle import shuffle_scale_moe
assert (
scale.dim() == 3 and scale.shape[-1] % 32 == 0 and scale.shape[-2] % 4 == 0
), f"GFX1250 scale swizzle needs (E,K_SCALE%4,N%32); got {tuple(scale.shape)}"
scale = shuffle_scale_moe(
scale, arch="gfx1250", preshuffle_factor=32, scale_kwidth=4
)
quant_tensor = convert_layout(
wrap_torch_tensor(quant_tensor, dtype=FP4), value_layout, **value_layout_opts
)
+77 -18
View File
@@ -225,6 +225,8 @@ direct_register_custom_op(
def use_aiter_triton_gemm(n, m, k, dtype):
from vllm.platforms.rocm import on_gfx1250
if (
not rocm_aiter_ops.is_triton_gemm_enabled()
# MI300's - fp8nuz=True
@@ -233,6 +235,9 @@ def use_aiter_triton_gemm(n, m, k, dtype):
):
return False
if on_gfx1250():
return True
# use hipblaslt for the larger GEMMs
if n > 2048 and m > 512:
return False
@@ -285,6 +290,44 @@ def wvsplitkrc_dispatch(n: int, k: int, m: int, cu_count: int) -> tuple[int, boo
return chunkk, fits
# On gfx11 a weight row stride that is a multiple of 2048 B makes the multi-row global
# loads of a GEMM collide on the L2/MALL channel hash. Offsetting the stride by one
# 128 B cache line dodges the collision and keeps each row cache-line aligned.
def cache_aliasing_pad_elems(k: int, element_size: int) -> int:
"""Row-stride pad, in elements, that stops an ``[n, k]`` weight aliasing on gfx11,
or 0 when the stride does not alias.
Args:
k: Number of elements per weight row.
element_size: Size of one element in bytes.
Returns:
The number of elements to add to the row stride.
"""
if (k * element_size) % 2048 != 0:
return 0
return 128 // element_size
def maybe_pad_weight_avoid_cache_aliasing(weight: torch.Tensor) -> torch.Tensor:
"""Offset a dense weight's row stride so it stops aliasing.
Returns a view with the original shape whose row stride is one cache line
wider, or ``weight`` unchanged when the pad does not apply.
"""
from vllm.platforms.rocm import on_gfx11
if not (current_platform.is_rocm() and on_gfx11()):
return weight
if weight.dim() != 2 or not weight.is_contiguous():
return weight
pad = cache_aliasing_pad_elems(weight.shape[1], weight.element_size())
if pad == 0:
return weight
return torch.nn.functional.pad(weight, (0, pad))[..., :-pad]
def rocm_unquantized_gemm_impl(
x: torch.Tensor, weight: torch.Tensor, bias: torch.Tensor | None = None
) -> torch.Tensor:
@@ -295,7 +338,7 @@ def rocm_unquantized_gemm_impl(
return linear_batch_invariant(x, weight, bias)
from vllm.platforms.rocm import on_gfx1x, on_gfx9, on_gfx950, on_gfx1250
from vllm.platforms.rocm import on_gfx1x, on_gfx9, on_gfx11, on_gfx950
n = x.numel() // x.size(-1)
m = weight.shape[0]
@@ -328,38 +371,54 @@ def rocm_unquantized_gemm_impl(
x_view = x.reshape(-1, x.size(-1)).contiguous()
return ops.wvSplitKrc(x_view, weight, cu_count, bias)
# gfx1250's aiter gemm_a16w16 uses the gluon backend, which requires
# K % 256 == 0 (it walks K with fixed-size descriptors and won't pad a
# partial last tile). Some whitelisted shapes have K=2880 (e.g. gpt-oss-120b
# hidden), so skip aiter there and fall back to the torch GEMM path below.
if use_aiter_triton_gemm(n, m, k, x.dtype) and not (on_gfx1250() and k % 256 != 0):
if use_aiter_triton_gemm(n, m, k, x.dtype):
from aiter.ops.triton.gemm_a16w16 import gemm_a16w16
return gemm_a16w16(x, weight, bias)
use_skinny = (
skinny_gemm_enabled = (
envs.VLLM_ROCM_USE_SKINNY_GEMM
and (on_gfx9() or on_gfx1x())
# build (gfx9/gfx11 ISA); fall back to torch GEMM there.
# TODO GFX1250: Include once skinny GEMM is supported on gfx1250
and x.dtype in [torch.float16, torch.bfloat16]
and k % 8 == 0
and skinny_operands_compatible
)
if use_skinny:
# wvSplitK allows row strides. Perf impact is only validated on gfx11.
weight_ok_for_wvsplitk = weight.is_contiguous() or (
on_gfx11() and weight.stride(1) == 1
)
use_wvsplitk = (
skinny_gemm_enabled
and weight_ok_for_wvsplitk
and (bias is None or bias.is_contiguous())
and (m == 1 or m > 8)
and 0 < n <= 5
)
use_llmm1 = (
skinny_gemm_enabled
and weight.is_contiguous()
and bias is None
and m % 4 == 0
and n == 1
and k <= 8192
)
if use_wvsplitk:
# The skinny kernels assume contiguous K elements. A shape-preserving
# reshape can retain a transposed activation's non-contiguous strides.
# Note: Only build that view inside the branches that consume it.
if (m == 1 or m > 8) and 0 < n <= 5:
x_view = x.reshape(-1, x.size(-1)).contiguous()
cu_count = num_compute_units()
out = ops.wvSplitK(weight, x_view, cu_count, bias)
return out.reshape(*x.shape[:-1], weight.shape[0])
elif m % 4 == 0 and n == 1 and k <= 8192 and bias is None:
x_view = x.reshape(-1, x.size(-1)).contiguous()
out = ops.LLMM1(weight, x_view, 4)
return out.reshape(*x.shape[:-1], weight.shape[0])
x_view = x.reshape(-1, x.size(-1)).contiguous()
out = ops.wvSplitK(weight, x_view, cu_count, bias)
return out.reshape(*x.shape[:-1], weight.shape[0])
if use_llmm1:
x_view = x.reshape(-1, x.size(-1)).contiguous()
out = ops.LLMM1(weight, x_view, 4)
return out.reshape(*x.shape[:-1], weight.shape[0])
if rocm_aiter_ops.is_tgemm_enabled():
from aiter.tuned_gemm import tgemm
@@ -607,6 +607,7 @@ class ParallelLMHead(VocabParallelEmbedding):
prefix: str = "",
*,
disable_tp: bool = False,
quant_method: QuantizeMethodBase | None = None,
):
self.has_bias = bias
super().__init__(
@@ -618,6 +619,7 @@ class ParallelLMHead(VocabParallelEmbedding):
quant_config,
prefix,
disable_tp=disable_tp,
quant_method=quant_method,
)
self.quant_config = quant_config
if bias: