Merge branch 'main' into patches
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -249,6 +249,8 @@ class FusedMoEQuantConfig:
|
||||
_w1: FusedMoEQuantDesc
|
||||
_w2: FusedMoEQuantDesc
|
||||
is_scale_swizzled: bool = True
|
||||
# Emit block-quant activation scales column-major (physical [G, M] layout).
|
||||
is_scale_transposed: bool = False
|
||||
|
||||
# MXFP4-specific TRTLLM parameters for SwiGLU activation clamping.
|
||||
# These correspond to gemm1_alpha, gemm1_beta, gemm1_clamp_limit
|
||||
@@ -503,6 +505,7 @@ class FusedMoEQuantConfig:
|
||||
w2_zp: torch.Tensor | None = None,
|
||||
weight_dtype: torch.dtype | str | None = None,
|
||||
is_scale_swizzled: bool = True,
|
||||
is_scale_transposed: bool = False,
|
||||
gemm1_alpha: float | None = None,
|
||||
gemm1_beta: float | None = None,
|
||||
gemm1_clamp_limit: float | None = None,
|
||||
@@ -537,6 +540,9 @@ class FusedMoEQuantConfig:
|
||||
- is_scale_swizzled: Whether the activation scale-factor layout is
|
||||
swizzled. Pass through to the underlying quantization kernel for
|
||||
dtypes that distinguish layouts (nvfp4, mxfp8). Defaults to True.
|
||||
- is_scale_transposed: Whether block-quantized activation scales are
|
||||
emitted column-major (physical [num_groups, num_tokens] layout) by
|
||||
the quantize step. Defaults to False.
|
||||
- gemm1_alpha: Optional MXFP4 TRTLLM SwiGLU alpha parameter.
|
||||
- gemm1_beta: Optional MXFP4 TRTLLM SwiGLU beta parameter.
|
||||
- gemm1_clamp_limit: Optional MXFP4 TRTLLM SwiGLU clamp limit.
|
||||
@@ -573,6 +579,7 @@ class FusedMoEQuantConfig:
|
||||
weight_dtype, w_shape, w2_scale, g2_alphas, w2_zp, w2_bias
|
||||
),
|
||||
is_scale_swizzled=is_scale_swizzled,
|
||||
is_scale_transposed=is_scale_transposed,
|
||||
gemm1_alpha=gemm1_alpha,
|
||||
gemm1_beta=gemm1_beta,
|
||||
gemm1_clamp_limit=gemm1_clamp_limit,
|
||||
@@ -597,6 +604,7 @@ def fp8_w8a8_moe_quant_config(
|
||||
a2_gscale: torch.Tensor | None = None,
|
||||
g1_alphas: torch.Tensor | None = None,
|
||||
g2_alphas: torch.Tensor | None = None,
|
||||
is_scale_transposed: bool = False,
|
||||
gemm1_alpha: float | None = None,
|
||||
gemm1_beta: float | None = None,
|
||||
gemm1_clamp_limit: float | None = None,
|
||||
@@ -617,6 +625,7 @@ def fp8_w8a8_moe_quant_config(
|
||||
per_act_token_quant=per_act_token_quant,
|
||||
per_out_ch_quant=per_out_ch_quant,
|
||||
block_shape=block_shape,
|
||||
is_scale_transposed=is_scale_transposed,
|
||||
gemm1_alpha=gemm1_alpha,
|
||||
gemm1_beta=gemm1_beta,
|
||||
gemm1_clamp_limit=gemm1_clamp_limit,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -16,6 +16,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kMxfp4Static,
|
||||
)
|
||||
from vllm.triton_utils import triton
|
||||
|
||||
__all__ = [
|
||||
"AiterW4A16ExpertsMonolithic",
|
||||
@@ -52,10 +53,10 @@ def _aiter_w4a16_silu_via_a8w4(
|
||||
from aiter.ops.triton.moe_op_gemm_a8w4 import moe_gemm_a8w4
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
|
||||
should_use_cdna4_mx_scale_swizzle,
|
||||
get_aiter_mx_scale_swizzle,
|
||||
)
|
||||
|
||||
swz = "CDNA4_SCALE" if should_use_cdna4_mx_scale_swizzle() else None
|
||||
swz = get_aiter_mx_scale_swizzle()
|
||||
quant_dtype = torch.float8_e4m3fn
|
||||
|
||||
g1_gammas = gammas if apply_router_weight_on_input else None
|
||||
@@ -134,6 +135,8 @@ def aiter_triton_kernel_w4a16_moe_forward(
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
score_mode: str | None = None,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
hash_indices_table: torch.Tensor | None = None,
|
||||
):
|
||||
assert quant_config is not None and rocm_aiter_ops.is_enabled()
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
|
||||
@@ -154,17 +157,37 @@ def aiter_triton_kernel_w4a16_moe_forward(
|
||||
_routing_mod.is_tdm_avail = lambda: False
|
||||
aiter_routing = _routing_mod.routing
|
||||
|
||||
if score_mode is not None:
|
||||
scale = routed_scaling_factor if routed_scaling_factor is not None else 1.0
|
||||
if hash_indices_table is not None:
|
||||
assert score_mode == "sqrtsoftplus" and input_ids is not None
|
||||
# Same block_m heuristic as aiter's routing().
|
||||
n_tokens, n_experts = gating_output.shape
|
||||
block_m = max(
|
||||
16, min(triton.next_power_of_2(max(1, n_tokens * topk // n_experts)), 128)
|
||||
)
|
||||
routing_data, gather_idx, scatter_idx = _routing_mod.routing_from_hash(
|
||||
gating_output,
|
||||
hash_indices_table,
|
||||
input_ids,
|
||||
topk,
|
||||
block_m,
|
||||
score_mode=score_mode,
|
||||
renorm=renormalize,
|
||||
routed_scaling_factor=scale,
|
||||
)
|
||||
elif score_mode is not None:
|
||||
use_grouped_topk = num_expert_group is not None and num_expert_group > 1
|
||||
routing_data, gather_idx, scatter_idx = aiter_routing(
|
||||
gating_output,
|
||||
topk,
|
||||
score_mode=score_mode,
|
||||
bias=e_score_correction_bias,
|
||||
renorm=renormalize,
|
||||
routed_scaling_factor=(
|
||||
routed_scaling_factor if routed_scaling_factor is not None else 1.0
|
||||
bias=(
|
||||
e_score_correction_bias.float()
|
||||
if e_score_correction_bias is not None
|
||||
else None
|
||||
),
|
||||
renorm=renormalize,
|
||||
routed_scaling_factor=scale,
|
||||
use_grouped_topk=use_grouped_topk,
|
||||
num_expert_group=num_expert_group,
|
||||
topk_group=topk_group,
|
||||
@@ -348,6 +371,10 @@ class AiterW4A16ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
|
||||
RoutingMethodType.DeepSeekV3,
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _supports_hash_routing() -> bool:
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _supports_router_logits_dtype(
|
||||
router_logits_dtype: torch.dtype | None,
|
||||
@@ -375,6 +402,8 @@ class AiterW4A16ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
routing_replay_out: torch.Tensor | None = None,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
hash_indices_table: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
routing_method = self.moe_config.routing_method
|
||||
if routing_method == RoutingMethodType.DeepseekV4:
|
||||
@@ -411,4 +440,6 @@ class AiterW4A16ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
score_mode=score_mode,
|
||||
input_ids=input_ids,
|
||||
hash_indices_table=hash_indices_table,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,373 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEParallelConfig,
|
||||
FusedMoEQuantConfig,
|
||||
RoutingMethodType,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kMxfp4Dynamic,
|
||||
kMxfp4Static,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"AiterW4A4ExpertsMonolithic",
|
||||
"aiter_triton_kernel_w4a4_moe_forward",
|
||||
]
|
||||
|
||||
|
||||
def _aiter_raw(t):
|
||||
if t is None or isinstance(t, torch.Tensor):
|
||||
return t
|
||||
return t.storage.data if hasattr(t, "storage") else t
|
||||
|
||||
|
||||
def aiter_triton_kernel_w4a4_moe_forward(
|
||||
hidden_states: torch.Tensor,
|
||||
w1,
|
||||
w2,
|
||||
gating_output: torch.Tensor,
|
||||
topk: int,
|
||||
renormalize: bool,
|
||||
activation: MoEActivation = MoEActivation.SWIGLUOAI,
|
||||
quant_config: FusedMoEQuantConfig | None = None,
|
||||
apply_router_weight_on_input: bool = False,
|
||||
global_num_experts: int = -1,
|
||||
expert_map: torch.Tensor | None = None,
|
||||
unpadded_N_w1=None,
|
||||
unpadded_K_w1=None,
|
||||
unpadded_N_w2=None,
|
||||
unpadded_K_w2=None,
|
||||
num_expert_group: int | None = None,
|
||||
topk_group: int | None = None,
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
score_mode: str | None = None,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
hash_indices_table: torch.Tensor | None = None,
|
||||
):
|
||||
assert quant_config is not None and rocm_aiter_ops.is_enabled()
|
||||
|
||||
from aiter.ops.triton.moe.moe_op_gemm_a4w4 import moe_gemm_a4w4, mxfp4_quant
|
||||
|
||||
try:
|
||||
from aiter.ops.triton.moe.moe_routing import routing as _routing_mod
|
||||
except ImportError:
|
||||
from aiter.ops.triton.moe_routing import routing as _routing_mod
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
|
||||
get_aiter_mx_scale_swizzle,
|
||||
)
|
||||
from vllm.platforms.rocm import on_gfx1250
|
||||
|
||||
if on_gfx1250():
|
||||
_routing_mod.is_tdm_avail = lambda: False
|
||||
aiter_routing = _routing_mod.routing
|
||||
|
||||
gating_output = torch.nan_to_num(gating_output, nan=0.0, posinf=0.0, neginf=0.0)
|
||||
|
||||
if hash_indices_table is not None:
|
||||
assert input_ids is not None, "hash routing requires input_ids"
|
||||
n_tokens, n_expts_tot = gating_output.shape
|
||||
tokens_per_expt = max(1, n_tokens * topk // n_expts_tot)
|
||||
block_m = max(16, min(1 << (tokens_per_expt - 1).bit_length(), 128))
|
||||
routing_data, gather_idx, scatter_idx = _routing_mod.routing_from_hash(
|
||||
gating_output,
|
||||
hash_indices_table,
|
||||
input_ids.to(hash_indices_table.dtype),
|
||||
topk,
|
||||
block_m,
|
||||
score_mode=score_mode or "sqrtsoftplus",
|
||||
renorm=renormalize,
|
||||
routed_scaling_factor=(
|
||||
routed_scaling_factor if routed_scaling_factor is not None else 1.0
|
||||
),
|
||||
)
|
||||
elif score_mode is not None:
|
||||
use_grouped_topk = num_expert_group is not None and num_expert_group > 1
|
||||
routing_data, gather_idx, scatter_idx = aiter_routing(
|
||||
gating_output,
|
||||
topk,
|
||||
score_mode=score_mode,
|
||||
bias=(
|
||||
e_score_correction_bias.float()
|
||||
if e_score_correction_bias is not None
|
||||
else None
|
||||
),
|
||||
renorm=renormalize,
|
||||
routed_scaling_factor=(
|
||||
routed_scaling_factor if routed_scaling_factor is not None else 1.0
|
||||
),
|
||||
use_grouped_topk=use_grouped_topk,
|
||||
num_expert_group=num_expert_group,
|
||||
topk_group=topk_group,
|
||||
)
|
||||
else:
|
||||
routing_data, gather_idx, scatter_idx = aiter_routing(
|
||||
gating_output, topk, sm_first=not renormalize
|
||||
)
|
||||
|
||||
if on_gfx1250():
|
||||
gather_src = gather_idx.to(torch.long) // topk
|
||||
hidden_states = hidden_states[gather_src]
|
||||
gather_idx = None
|
||||
|
||||
w1_data = _aiter_raw(w1)
|
||||
w2_data = _aiter_raw(w2)
|
||||
w1_scale = _aiter_raw(quant_config.w1_scale)
|
||||
w2_scale = _aiter_raw(quant_config.w2_scale)
|
||||
|
||||
gammas = routing_data.gate_scal if routing_data else None
|
||||
|
||||
swiglu_limit = (
|
||||
quant_config.gemm1_clamp_limit
|
||||
if quant_config.gemm1_clamp_limit is not None
|
||||
else 7.0
|
||||
)
|
||||
# SWIGLUOAI needs the alpha/residual swiglu fused into the GEMM1 epilogue,
|
||||
# which reads gate/up interleaved along N (see the oracle's weight prep).
|
||||
fused_swiglu = activation in (
|
||||
MoEActivation.SWIGLUOAI,
|
||||
MoEActivation.SWIGLUOAI_UNINTERLEAVE,
|
||||
)
|
||||
swiglu_kwargs: dict[str, object] = {}
|
||||
if fused_swiglu:
|
||||
assert quant_config.gemm1_beta in (None, 1.0), (
|
||||
"aiter's fused swiglu hardcodes the residual to (up + 1); "
|
||||
f"gemm1_beta={quant_config.gemm1_beta} cannot be expressed"
|
||||
)
|
||||
swiglu_kwargs = {
|
||||
"alpha": (
|
||||
quant_config.gemm1_alpha
|
||||
if quant_config.gemm1_alpha is not None
|
||||
else 1.0
|
||||
),
|
||||
"limit": swiglu_limit,
|
||||
"swiglu_add_residual": quant_config.gemm1_beta == 1.0,
|
||||
}
|
||||
swizzle_mx_scale = get_aiter_mx_scale_swizzle()
|
||||
|
||||
x_q, x_scale = mxfp4_quant(hidden_states.to(torch.bfloat16))
|
||||
|
||||
# GEMM1: gate+up projection.
|
||||
raw_intermediate = moe_gemm_a4w4(
|
||||
x_q,
|
||||
w1_data,
|
||||
x_scale,
|
||||
w1_scale,
|
||||
bias=quant_config.w1_bias,
|
||||
routing_data=routing_data,
|
||||
gather_indx=gather_idx,
|
||||
gammas=gammas if apply_router_weight_on_input else None,
|
||||
swizzle_mx_scale=swizzle_mx_scale,
|
||||
apply_swiglu=fused_swiglu,
|
||||
**swiglu_kwargs,
|
||||
)
|
||||
|
||||
if fused_swiglu:
|
||||
# The swiglu epilogue already halved N.
|
||||
intermediate = (
|
||||
raw_intermediate
|
||||
if unpadded_N_w1 is None
|
||||
else raw_intermediate[:, : unpadded_N_w1 // 2]
|
||||
)
|
||||
else:
|
||||
if unpadded_N_w1 is not None:
|
||||
raw_intermediate = raw_intermediate[:, :unpadded_N_w1]
|
||||
|
||||
# SiLU(gate) * up on the concatenated [gate | up] halves
|
||||
from aiter.ops.triton.fusions.fused_clamp_act_mul import fused_clamp_act_mul
|
||||
|
||||
half_n = raw_intermediate.shape[-1] // 2
|
||||
intermediate = torch.empty(
|
||||
raw_intermediate.shape[0],
|
||||
half_n,
|
||||
dtype=raw_intermediate.dtype,
|
||||
device=raw_intermediate.device,
|
||||
)
|
||||
fused_clamp_act_mul(
|
||||
raw_intermediate,
|
||||
out=intermediate,
|
||||
swiglu_limit=swiglu_limit,
|
||||
activation="silu",
|
||||
dtype_quant=None,
|
||||
)
|
||||
|
||||
# GEMM2: down projection with scatter-reduce
|
||||
mid_q, mid_scale = mxfp4_quant(intermediate.to(torch.bfloat16))
|
||||
|
||||
out = moe_gemm_a4w4(
|
||||
mid_q,
|
||||
w2_data,
|
||||
mid_scale,
|
||||
w2_scale,
|
||||
bias=quant_config.w2_bias,
|
||||
routing_data=routing_data,
|
||||
scatter_indx=scatter_idx,
|
||||
gammas=None if apply_router_weight_on_input else gammas,
|
||||
swizzle_mx_scale=swizzle_mx_scale,
|
||||
apply_swiglu=False,
|
||||
)
|
||||
|
||||
return out
|
||||
|
||||
|
||||
class AiterW4A4ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
|
||||
"""Monolithic MXFP4 W4A4 expert using AITER Triton/Gluon kernels (gfx1250).
|
||||
|
||||
Uses moe_gemm_a4w4 (auto-selects gluon backend on gfx1250) with dynamic
|
||||
MXFP4 activation quantization via mxfp4_quant.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
moe_config: FusedMoEConfig,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
):
|
||||
super().__init__(moe_config, quant_config)
|
||||
self.topk = moe_config.experts_per_token
|
||||
self.renormalize = moe_config.routing_method in (
|
||||
RoutingMethodType.Renormalize,
|
||||
RoutingMethodType.RenormalizeNaive,
|
||||
RoutingMethodType.DeepseekV4,
|
||||
RoutingMethodType.DeepSeekV3,
|
||||
RoutingMethodType.MiniMax2,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def activation_format() -> mk.FusedMoEActivationFormat:
|
||||
return mk.FusedMoEActivationFormat.Standard
|
||||
|
||||
@staticmethod
|
||||
def _supports_hash_routing() -> bool:
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _supports_current_device() -> bool:
|
||||
if not rocm_aiter_ops.is_enabled():
|
||||
return False
|
||||
from vllm.platforms.rocm import on_gfx1250
|
||||
|
||||
return on_gfx1250()
|
||||
|
||||
@staticmethod
|
||||
def _supports_no_act_and_mul() -> bool:
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _supports_quant_scheme(
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> bool:
|
||||
return (weight_key, activation_key) == (kMxfp4Static, kMxfp4Dynamic)
|
||||
|
||||
@staticmethod
|
||||
def _supports_activation(activation: MoEActivation) -> bool:
|
||||
return activation in (
|
||||
MoEActivation.SWIGLUOAI,
|
||||
MoEActivation.SWIGLUOAI_UNINTERLEAVE,
|
||||
MoEActivation.SILU,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _supports_parallel_config(
|
||||
moe_parallel_config: FusedMoEParallelConfig,
|
||||
) -> bool:
|
||||
return (
|
||||
not moe_parallel_config.use_all2all_kernels
|
||||
and not moe_parallel_config.enable_eplb
|
||||
and moe_parallel_config.dp_size <= 1
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _supports_routing_method(
|
||||
routing_method: RoutingMethodType,
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> bool:
|
||||
return routing_method in [
|
||||
RoutingMethodType.Renormalize,
|
||||
RoutingMethodType.RenormalizeNaive,
|
||||
RoutingMethodType.DeepseekV4,
|
||||
RoutingMethodType.DeepSeekV3,
|
||||
RoutingMethodType.MiniMax2,
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _supports_router_logits_dtype(
|
||||
router_logits_dtype: torch.dtype | None,
|
||||
routing_method: RoutingMethodType,
|
||||
) -> bool:
|
||||
return True
|
||||
|
||||
@property
|
||||
def expects_unquantized_inputs(self) -> bool:
|
||||
return True
|
||||
|
||||
def apply(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
activation: MoEActivation,
|
||||
global_num_experts: int,
|
||||
expert_map: torch.Tensor | None,
|
||||
a1q_scale: torch.Tensor | None,
|
||||
apply_router_weight_on_input: bool,
|
||||
num_expert_group: int | None = None,
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
routing_replay_out: torch.Tensor | None = None,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
hash_indices_table: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
assert self.moe_config.intermediate_size_per_partition_unpadded is not None
|
||||
assert self.moe_config.hidden_dim_unpadded is not None
|
||||
routing_method = self.moe_config.routing_method
|
||||
if routing_method == RoutingMethodType.DeepseekV4:
|
||||
score_mode = "sqrtsoftplus"
|
||||
elif routing_method in (
|
||||
RoutingMethodType.DeepSeekV3,
|
||||
RoutingMethodType.MiniMax2,
|
||||
):
|
||||
# Sigmoid scores + routing bias; grouped for DeepSeek-R1/V3,
|
||||
# ungrouped for MiniMax-M3.
|
||||
score_mode = "sigmoid"
|
||||
else:
|
||||
score_mode = None
|
||||
return aiter_triton_kernel_w4a4_moe_forward(
|
||||
hidden_states=hidden_states,
|
||||
w1=w1,
|
||||
w2=w2,
|
||||
gating_output=router_logits,
|
||||
topk=self.topk,
|
||||
renormalize=self.renormalize,
|
||||
activation=activation,
|
||||
global_num_experts=global_num_experts,
|
||||
expert_map=expert_map,
|
||||
quant_config=self.quant_config,
|
||||
apply_router_weight_on_input=apply_router_weight_on_input,
|
||||
unpadded_N_w1=(
|
||||
self.moe_config.intermediate_size_per_partition_unpadded * 2
|
||||
),
|
||||
unpadded_K_w1=self.moe_config.hidden_dim_unpadded,
|
||||
unpadded_N_w2=self.moe_config.hidden_dim_unpadded,
|
||||
unpadded_K_w2=(self.moe_config.intermediate_size_per_partition_unpadded),
|
||||
num_expert_group=num_expert_group,
|
||||
topk_group=topk_group,
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
score_mode=score_mode,
|
||||
input_ids=input_ids,
|
||||
hash_indices_table=hash_indices_table,
|
||||
)
|
||||
@@ -61,12 +61,6 @@ def aiter_triton_kernel_w4a8_moe_forward(
|
||||
gating_output, topk, sm_first=not renormalize
|
||||
)
|
||||
|
||||
# gfx1250: aiter's in-kernel gather is numerically broken
|
||||
if on_gfx1250():
|
||||
gather_src = gather_idx.to(torch.long) // topk
|
||||
hidden_states = hidden_states[gather_src]
|
||||
gather_idx = None
|
||||
|
||||
return triton_kernel_fused_mxfp4_w4a8_experts(
|
||||
None,
|
||||
hidden_states,
|
||||
@@ -130,11 +124,11 @@ def triton_kernel_fused_mxfp4_w4a8_experts(
|
||||
from aiter.ops.triton.quant_moe import downcast_to_static_fp8
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
|
||||
should_use_cdna4_mx_scale_swizzle,
|
||||
get_aiter_mx_scale_swizzle,
|
||||
weight_mx_scale,
|
||||
)
|
||||
|
||||
_swizzle_mx_scale = "CDNA4_SCALE" if should_use_cdna4_mx_scale_swizzle() else None
|
||||
_swizzle_mx_scale = get_aiter_mx_scale_swizzle()
|
||||
|
||||
assert quant_config.w1_precision is not None, (
|
||||
"w1_precision in quant config can't be None"
|
||||
|
||||
@@ -436,6 +436,7 @@ def FusedMoEFactory(
|
||||
# TODO get from router? needs to be truncated?
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
apply_router_weight_on_input=apply_router_weight_on_input,
|
||||
hash_indices_table=hash_indices_table,
|
||||
**routed_experts_args if routed_experts_args is not None else {},
|
||||
)
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@ from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from math import prod
|
||||
from typing import final
|
||||
from typing import Any, final
|
||||
|
||||
import torch
|
||||
|
||||
@@ -579,7 +579,11 @@ class FusedMoEExperts(ABC):
|
||||
return False, _make_reason(
|
||||
f"parallel config {moe_config.moe_parallel_config}"
|
||||
)
|
||||
elif moe_config.has_hash_routing and cls.is_monolithic():
|
||||
elif (
|
||||
moe_config.has_hash_routing
|
||||
and cls.is_monolithic()
|
||||
and not cls._supports_hash_routing()
|
||||
):
|
||||
return False, _make_reason("hash routing")
|
||||
elif moe_config.require_decomposed_backend and cls.is_monolithic():
|
||||
return False, _make_reason("routes adjusted after routing")
|
||||
@@ -659,6 +663,15 @@ class FusedMoEExperts(ABC):
|
||||
"""
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _supports_hash_routing() -> bool:
|
||||
"""Whether a monolithic kernel implements hash routing itself.
|
||||
|
||||
Kernels that return True receive `input_ids` and `hash_indices_table`
|
||||
in `apply()` for layers with a hash-routing table.
|
||||
"""
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _supports_router_logits_dtype(
|
||||
router_logits_dtype: torch.dtype | None,
|
||||
@@ -1584,11 +1597,20 @@ class FusedMoEKernelMonolithicImpl:
|
||||
topk_group: int | None = None,
|
||||
*,
|
||||
routing_sink: RoutedExpertsSink | None,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
hash_indices_table: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
"""Same as forward(), except uses router_logits as opposed
|
||||
to the topk_ids and topk_weights. This is used for kernels
|
||||
that have fused router + experts (e.g. FLASHINFER_TRTLLM).
|
||||
"""
|
||||
# Only experts with _supports_hash_routing() accept these.
|
||||
hash_kwargs: dict[str, Any] = {}
|
||||
if hash_indices_table is not None:
|
||||
hash_kwargs = {
|
||||
"input_ids": input_ids,
|
||||
"hash_indices_table": hash_indices_table,
|
||||
}
|
||||
a1q, a1q_scale, router_logits = self.prepare_finalize.prepare(
|
||||
hidden_states,
|
||||
router_logits=router_logits,
|
||||
@@ -1605,6 +1627,11 @@ class FusedMoEKernelMonolithicImpl:
|
||||
f"kernel {type(self.fused_experts).__name__}."
|
||||
)
|
||||
routing_replay_out = routing_sink.buffer[: len(a1q)]
|
||||
hash_kwargs = (
|
||||
{"input_ids": input_ids, "hash_indices_table": hash_indices_table}
|
||||
if hash_indices_table is not None
|
||||
else {}
|
||||
)
|
||||
fused_out = self.fused_experts.apply(
|
||||
hidden_states=a1q,
|
||||
w1=w1,
|
||||
@@ -1621,6 +1648,7 @@ class FusedMoEKernelMonolithicImpl:
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
topk_group=topk_group,
|
||||
routing_replay_out=routing_replay_out,
|
||||
**hash_kwargs,
|
||||
)
|
||||
if routing_sink is not None:
|
||||
routing_sink.capture_fn(routing_replay_out)
|
||||
@@ -1733,6 +1761,8 @@ class FusedMoEKernel:
|
||||
topk_group: int | None = None,
|
||||
*,
|
||||
routing_sink: RoutedExpertsSink | None,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
hash_indices_table: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
assert isinstance(self.impl, FusedMoEKernelMonolithicImpl)
|
||||
return self.impl.apply(
|
||||
@@ -1749,6 +1779,8 @@ class FusedMoEKernel:
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
topk_group=topk_group,
|
||||
routing_sink=routing_sink,
|
||||
input_ids=input_ids,
|
||||
hash_indices_table=hash_indices_table,
|
||||
)
|
||||
|
||||
def apply(
|
||||
|
||||
@@ -841,6 +841,11 @@ def make_fp8_moe_quant_config(
|
||||
block_shape=block_shape,
|
||||
per_act_token_quant=per_act_token_quant,
|
||||
per_out_ch_quant=per_out_ch_quant,
|
||||
# TRTLLM fp8-block consumes activation scales as [K/128, M]; emit them
|
||||
# transposed from the quant kernel instead of copying afterwards.
|
||||
is_scale_transposed=(
|
||||
fp8_backend == Fp8MoeBackend.FLASHINFER_TRTLLM and block_shape == [128, 128]
|
||||
),
|
||||
gemm1_alpha=gemm1_alpha,
|
||||
gemm1_beta=gemm1_beta,
|
||||
gemm1_clamp_limit=swiglu_limit,
|
||||
|
||||
@@ -152,7 +152,9 @@ class Mxfp4MoeBackend(Enum):
|
||||
# W4A16: aiter Triton moe_gemm_a16w4 kernel (gfx942/gfx950/gfx1250)
|
||||
AITER_TRITON_MXFP4_BF16 = "AITER_TRITON_MXFP4_BF16"
|
||||
AITER_MXFP4_FP8 = "AITER_MXFP4_FP8" # W4A8: triton kernel
|
||||
AITER_MXFP4_MXFP4 = "AITER_MXFP4_MXFP4" # W4A4: CK kernel
|
||||
AITER_MXFP4_MXFP4 = "AITER_MXFP4_MXFP4" # W4A4: CK kernel (gfx950)
|
||||
# W4A4: aiter Triton/Gluon moe_gemm_a4w4 kernel (gfx1250)
|
||||
AITER_TRITON_MXFP4_MXFP4 = "AITER_TRITON_MXFP4_MXFP4"
|
||||
# Triton
|
||||
TRITON = "TRITON"
|
||||
TRITON_UNFUSED = "TRITON_UNFUSED"
|
||||
@@ -310,6 +312,13 @@ def backend_to_kernel_cls(
|
||||
|
||||
return [AiterMxfp4Experts]
|
||||
|
||||
elif backend == Mxfp4MoeBackend.AITER_TRITON_MXFP4_MXFP4:
|
||||
from vllm.model_executor.layers.fused_moe.experts.aiter_mxfp4_w4a4_moe import (
|
||||
AiterW4A4ExpertsMonolithic,
|
||||
)
|
||||
|
||||
return [AiterW4A4ExpertsMonolithic]
|
||||
|
||||
elif backend == Mxfp4MoeBackend.XPU:
|
||||
from vllm.model_executor.layers.fused_moe.experts.xpu_moe import XPUExpertsMxFp4
|
||||
|
||||
@@ -370,9 +379,11 @@ def map_mxfp4_backend(runner_backend: MoEBackend) -> list[Mxfp4MoeBackend]:
|
||||
Mxfp4MoeBackend.AITER_TRITON_MXFP4_BF16,
|
||||
Mxfp4MoeBackend.AITER_MXFP4_FP8,
|
||||
Mxfp4MoeBackend.AITER_MXFP4_MXFP4,
|
||||
Mxfp4MoeBackend.AITER_TRITON_MXFP4_MXFP4,
|
||||
],
|
||||
"aiter_triton_mxfp4_bf16": [Mxfp4MoeBackend.AITER_TRITON_MXFP4_BF16],
|
||||
"aiter_mxfp4_fp8": [Mxfp4MoeBackend.AITER_MXFP4_FP8],
|
||||
"aiter_triton_mxfp4_mxfp4": [Mxfp4MoeBackend.AITER_TRITON_MXFP4_MXFP4],
|
||||
"aiter_mxfp4_mxfp4": [Mxfp4MoeBackend.AITER_MXFP4_MXFP4],
|
||||
"xpu": [Mxfp4MoeBackend.XPU],
|
||||
"cpu": [Mxfp4MoeBackend.CPU],
|
||||
@@ -464,6 +475,7 @@ def _backend_activation_key(backend: Mxfp4MoeBackend) -> QuantKey | None:
|
||||
return kFp8StaticTensorSym
|
||||
if backend in (
|
||||
Mxfp4MoeBackend.AITER_MXFP4_MXFP4,
|
||||
Mxfp4MoeBackend.AITER_TRITON_MXFP4_MXFP4,
|
||||
Mxfp4MoeBackend.CUTLASS_MXFP4_MXFP4,
|
||||
):
|
||||
return kMxfp4Dynamic
|
||||
@@ -896,6 +908,18 @@ def mxfp4_round_up_hidden_size_and_intermediate_size(
|
||||
return hidden_size, intermediate_size
|
||||
|
||||
|
||||
def _interleave_gate_up_rows(t: torch.Tensor) -> torch.Tensor:
|
||||
"""Repack [E, 2n, X] from [all gates | all ups] to [g0, u0, g1, u1, ...]."""
|
||||
num_experts, two_n, inner = t.shape
|
||||
assert two_n % 2 == 0, f"expected an even gate/up row count, got {two_n}"
|
||||
return (
|
||||
t.view(num_experts, 2, two_n // 2, inner)
|
||||
.transpose(1, 2)
|
||||
.contiguous()
|
||||
.view(num_experts, two_n, inner)
|
||||
)
|
||||
|
||||
|
||||
def convert_gpt_oss_weight_to_mxfp4_moe_kernel_format(
|
||||
mxfp4_backend: Mxfp4MoeBackend,
|
||||
layer: torch.nn.Module,
|
||||
@@ -1192,9 +1216,56 @@ def convert_gpt_oss_weight_to_mxfp4_moe_kernel_format(
|
||||
w2_bias,
|
||||
)
|
||||
|
||||
elif mxfp4_backend == Mxfp4MoeBackend.AITER_MXFP4_MXFP4:
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
elif mxfp4_backend == Mxfp4MoeBackend.AITER_TRITON_MXFP4_MXFP4:
|
||||
if w13_bias is not None:
|
||||
w13_bias = w13_bias.data.to(torch.float32)
|
||||
if w2_bias is not None:
|
||||
w2_bias = w2_bias.data.to(torch.float32)
|
||||
|
||||
# Triton moe_gemm_a4w4: uint8 column-major weights.
|
||||
w13_data = w13_weight.data.view(torch.uint8)
|
||||
w2_data = w2_weight.data.view(torch.uint8)
|
||||
w13_scale_data = w13_weight_scale.data.view(torch.uint8)
|
||||
|
||||
# SILU applies the activation outside the GEMM and keeps the
|
||||
# checkpoint's packed halves; the SWIGLUOAI epilogue needs them
|
||||
# interleaved.
|
||||
if getattr(layer, "activation", None) in (
|
||||
MoEActivation.SWIGLUOAI,
|
||||
MoEActivation.SWIGLUOAI_UNINTERLEAVE,
|
||||
):
|
||||
w13_data = _interleave_gate_up_rows(w13_data)
|
||||
w13_scale_data = _interleave_gate_up_rows(w13_scale_data)
|
||||
|
||||
# Transpose to [E, K, N] — the view has stride(-2)==1 (column-major),
|
||||
# which is what moe_gemm_a4w4 asserts.
|
||||
w13_data = w13_data.transpose(1, 2)
|
||||
w2_data = w2_data.transpose(1, 2)
|
||||
|
||||
# Scales are [E, N, K_scale]; both kernels index them as
|
||||
# [E, K_scale, N] with K innermost, so transpose(1, 2)
|
||||
# Keep them uint8: Triton moe_gemm_a4w4 takes e8m0 scales
|
||||
w13_scale = w13_scale_data.transpose(1, 2)
|
||||
w2_scale = w2_weight_scale.data.view(torch.uint8).transpose(1, 2)
|
||||
|
||||
from aiter.ops.triton.utils.shuffle import shuffle_scale_moe
|
||||
|
||||
w13_scale = shuffle_scale_moe(
|
||||
w13_scale,
|
||||
arch="gfx1250",
|
||||
preshuffle_factor=32,
|
||||
scale_kwidth=4,
|
||||
)
|
||||
w2_scale = shuffle_scale_moe(
|
||||
w2_scale,
|
||||
arch="gfx1250",
|
||||
preshuffle_factor=32,
|
||||
scale_kwidth=4,
|
||||
)
|
||||
|
||||
return (w13_data, w2_data, w13_scale, w2_scale, w13_bias, w2_bias)
|
||||
|
||||
elif mxfp4_backend == Mxfp4MoeBackend.AITER_MXFP4_MXFP4:
|
||||
if w13_bias is not None:
|
||||
w13_bias = w13_bias.data.to(torch.float32)
|
||||
if w2_bias is not None:
|
||||
@@ -1203,6 +1274,8 @@ def convert_gpt_oss_weight_to_mxfp4_moe_kernel_format(
|
||||
# e8m0_shuffle on weight scales (GFX950 swizzle layout)
|
||||
from aiter.utility.fp4_utils import e8m0_shuffle
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
s0, s1, _ = w13_weight_scale.shape
|
||||
w13_weight_scale.data = e8m0_shuffle(w13_weight_scale.view(s0 * s1, -1)).view(
|
||||
s0, s1, -1
|
||||
@@ -1935,6 +2008,7 @@ def convert_weight_to_mxfp4_moe_kernel_format(
|
||||
)
|
||||
elif mxfp4_backend in (
|
||||
Mxfp4MoeBackend.AITER_MXFP4_MXFP4,
|
||||
Mxfp4MoeBackend.AITER_TRITON_MXFP4_MXFP4,
|
||||
Mxfp4MoeBackend.AITER_MXFP4_FP8,
|
||||
):
|
||||
return convert_gpt_oss_weight_to_mxfp4_moe_kernel_format(
|
||||
@@ -2088,6 +2162,7 @@ def make_mxfp4_moe_quant_config(
|
||||
)
|
||||
elif mxfp4_backend in (
|
||||
Mxfp4MoeBackend.AITER_MXFP4_MXFP4,
|
||||
Mxfp4MoeBackend.AITER_TRITON_MXFP4_MXFP4,
|
||||
Mxfp4MoeBackend.CUTLASS_MXFP4_MXFP4,
|
||||
):
|
||||
return ocp_mx_moe_quant_config(
|
||||
|
||||
@@ -31,6 +31,7 @@ def _quantize_input(
|
||||
per_act_token_quant=quant_config.per_act_token_quant,
|
||||
block_shape=quant_config.block_shape,
|
||||
is_scale_swizzled=quant_config.is_scale_swizzled,
|
||||
is_scale_transposed=quant_config.is_scale_transposed,
|
||||
mx_alignment=quant_config.mx_alignment,
|
||||
)
|
||||
|
||||
|
||||
@@ -0,0 +1,682 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Triton input-staging kernels for DeepGEMM MegaMoE.
|
||||
|
||||
Quantize hidden states to fp8 with E8M0 group scales, to NVFP4 with per-token
|
||||
global scales, or copy them as BF16, and repack the routing top-k tensors into
|
||||
the int64/float32 layout that the DeepGEMM MegaMoE kernels consume.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
FP8_DTYPE,
|
||||
QuantKey,
|
||||
kMxfp8Dynamic,
|
||||
)
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
_E8M0_SCALE_DTYPES = (torch.uint8, torch.float8_e8m0fnu)
|
||||
# Inverse of the NVFP4 range: E2M1 max (6) times E4M3 max (448).
|
||||
_INV_NVFP4_MAX = tl.constexpr(1.0 / 2688.0)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _quantize_fp8_groups(
|
||||
hidden,
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_K: tl.constexpr,
|
||||
GROUP_K: tl.constexpr,
|
||||
USE_UE8M0: tl.constexpr,
|
||||
):
|
||||
"""Return fp8 values, fp32 group scales and, for UE8M0, the scale exponents."""
|
||||
num_groups: tl.constexpr = BLOCK_K // GROUP_K
|
||||
hidden_groups = tl.reshape(tl.abs(hidden), [BLOCK_M, num_groups, GROUP_K])
|
||||
amax = tl.max(hidden_groups, axis=2)
|
||||
amax = tl.maximum(amax, 1.0e-4)
|
||||
|
||||
scale = amax / 448.0
|
||||
if USE_UE8M0:
|
||||
scale_bits = scale.to(tl.uint32, bitcast=True)
|
||||
scale_exp = ((scale_bits >> 23) & 0xFF) + ((scale_bits & 0x7FFFFF) != 0).to(
|
||||
tl.uint32
|
||||
)
|
||||
scale_exp = tl.minimum(tl.maximum(scale_exp, 1), 254)
|
||||
quant_scale = (scale_exp << 23).to(tl.float32, bitcast=True)
|
||||
else:
|
||||
scale_exp = scale.to(tl.uint32, bitcast=True)
|
||||
quant_scale = scale
|
||||
|
||||
hidden_groups = tl.reshape(hidden, [BLOCK_M, num_groups, GROUP_K])
|
||||
scaled = hidden_groups * (1.0 / quant_scale)[:, :, None]
|
||||
scaled = tl.reshape(scaled, [BLOCK_M, BLOCK_K])
|
||||
return scaled.to(tl.float8e4nv), quant_scale, scale_exp
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _pack_bytes(values, NUM_BYTES: tl.constexpr):
|
||||
"""Pack the last dim of ``[..., NUM_BYTES]`` byte values into int32 words."""
|
||||
offsets = tl.arange(0, NUM_BYTES)
|
||||
return tl.sum(values.to(tl.uint32) << (offsets * 8), axis=-1).to(tl.int32)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _shared_sf_row(token_id, SHARED_BLOCK_M: tl.constexpr):
|
||||
"""Row of ``token_id`` in the shared-expert MN-major UTCCP scale pages."""
|
||||
m_block_id = token_id // SHARED_BLOCK_M
|
||||
m_in_block = token_id % SHARED_BLOCK_M
|
||||
aligned_block_m: tl.constexpr = triton.cdiv(SHARED_BLOCK_M, 128) * 128
|
||||
transposed_m = (
|
||||
(m_in_block // 128) * 128 + (m_in_block % 32) * 4 + (m_in_block % 128) // 32
|
||||
)
|
||||
return m_block_id * aligned_block_m + transposed_m
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _store_topk(
|
||||
token_id,
|
||||
token_mask,
|
||||
topk_ids,
|
||||
topk_weights,
|
||||
is_padding,
|
||||
topk_idx_out,
|
||||
topk_weights_out,
|
||||
topk_ids_stride_m: tl.constexpr,
|
||||
topk_ids_stride_k: tl.constexpr,
|
||||
topk_weights_stride_m: tl.constexpr,
|
||||
topk_weights_stride_k: tl.constexpr,
|
||||
is_padding_stride_m: tl.constexpr,
|
||||
topk_idx_stride_m: tl.constexpr,
|
||||
topk_idx_stride_k: tl.constexpr,
|
||||
topk_weights_out_stride_m: tl.constexpr,
|
||||
topk_weights_out_stride_k: tl.constexpr,
|
||||
top_k: tl.constexpr,
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_TOPK: tl.constexpr,
|
||||
) -> None:
|
||||
"""Repack top-k routing as int64 ids and fp32 weights; padding is unrouted."""
|
||||
topk_offsets = tl.arange(0, BLOCK_TOPK)
|
||||
topk_mask = token_mask[:, None] & (topk_offsets[None, :] < top_k)
|
||||
token_is_padding = tl.full((BLOCK_M,), False, tl.int1)
|
||||
if is_padding is not None:
|
||||
token_is_padding = tl.load(
|
||||
is_padding + token_id * is_padding_stride_m,
|
||||
mask=token_mask,
|
||||
other=True,
|
||||
)
|
||||
|
||||
ids = tl.load(
|
||||
topk_ids
|
||||
+ token_id[:, None] * topk_ids_stride_m
|
||||
+ topk_offsets[None, :] * topk_ids_stride_k,
|
||||
mask=topk_mask,
|
||||
other=0,
|
||||
).to(tl.int64)
|
||||
ids = tl.where(token_is_padding[:, None], -1, ids)
|
||||
tl.store(
|
||||
topk_idx_out
|
||||
+ token_id[:, None] * topk_idx_stride_m
|
||||
+ topk_offsets[None, :] * topk_idx_stride_k,
|
||||
ids,
|
||||
mask=topk_mask,
|
||||
)
|
||||
|
||||
weights = tl.load(
|
||||
topk_weights
|
||||
+ token_id[:, None] * topk_weights_stride_m
|
||||
+ topk_offsets[None, :] * topk_weights_stride_k,
|
||||
mask=topk_mask,
|
||||
other=0.0,
|
||||
)
|
||||
weights = tl.where(token_is_padding[:, None], 0.0, weights)
|
||||
tl.store(
|
||||
topk_weights_out
|
||||
+ token_id[:, None] * topk_weights_out_stride_m
|
||||
+ topk_offsets[None, :] * topk_weights_out_stride_k,
|
||||
weights,
|
||||
mask=topk_mask,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["num_tokens"])
|
||||
def _prepare_megamoe_inputs_kernel(
|
||||
hidden_states,
|
||||
x_out,
|
||||
x_sf,
|
||||
shared_x_sf,
|
||||
topk_ids,
|
||||
topk_weights,
|
||||
is_padding,
|
||||
topk_idx_out,
|
||||
topk_weights_out,
|
||||
hidden_stride_m: tl.constexpr,
|
||||
hidden_stride_k: tl.constexpr,
|
||||
x_stride_m: tl.constexpr,
|
||||
x_stride_k: tl.constexpr,
|
||||
x_sf_stride_m: tl.constexpr,
|
||||
x_sf_stride_k: tl.constexpr,
|
||||
shared_x_sf_stride_m: tl.constexpr,
|
||||
shared_x_sf_stride_k: tl.constexpr,
|
||||
topk_ids_stride_m: tl.constexpr,
|
||||
topk_ids_stride_k: tl.constexpr,
|
||||
topk_weights_stride_m: tl.constexpr,
|
||||
topk_weights_stride_k: tl.constexpr,
|
||||
is_padding_stride_m: tl.constexpr,
|
||||
topk_idx_stride_m: tl.constexpr,
|
||||
topk_idx_stride_k: tl.constexpr,
|
||||
topk_weights_out_stride_m: tl.constexpr,
|
||||
topk_weights_out_stride_k: tl.constexpr,
|
||||
num_tokens,
|
||||
hidden_size: tl.constexpr,
|
||||
top_k: tl.constexpr,
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_K: tl.constexpr,
|
||||
GROUP_K: tl.constexpr,
|
||||
BLOCK_TOPK: tl.constexpr,
|
||||
SHARED_BLOCK_M: tl.constexpr,
|
||||
QUANTIZE: tl.constexpr,
|
||||
USE_UE8M0: tl.constexpr,
|
||||
) -> None:
|
||||
token_id = tl.program_id(0) * BLOCK_M + tl.arange(0, BLOCK_M)
|
||||
token_mask = token_id < num_tokens
|
||||
k_block_id = tl.program_id(1)
|
||||
|
||||
k_offsets = k_block_id * BLOCK_K + tl.arange(0, BLOCK_K)
|
||||
k_mask = token_mask[:, None] & (k_offsets[None, :] < hidden_size)
|
||||
hidden = tl.load(
|
||||
hidden_states
|
||||
+ token_id[:, None] * hidden_stride_m
|
||||
+ k_offsets[None, :] * hidden_stride_k,
|
||||
mask=k_mask,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
|
||||
if not QUANTIZE:
|
||||
tl.store(
|
||||
x_out + token_id[:, None] * x_stride_m + k_offsets[None, :] * x_stride_k,
|
||||
hidden.to(x_out.dtype.element_ty),
|
||||
mask=k_mask,
|
||||
)
|
||||
else:
|
||||
num_groups: tl.constexpr = BLOCK_K // GROUP_K
|
||||
fp8, quant_scale, scale_exp = _quantize_fp8_groups(
|
||||
hidden, BLOCK_M, BLOCK_K, GROUP_K, USE_UE8M0
|
||||
)
|
||||
tl.store(
|
||||
x_out + token_id[:, None] * x_stride_m + k_offsets[None, :] * x_stride_k,
|
||||
fp8,
|
||||
mask=k_mask,
|
||||
)
|
||||
|
||||
if USE_UE8M0:
|
||||
packed_scale = _pack_bytes(scale_exp, num_groups)
|
||||
tl.store(
|
||||
x_sf + token_id * x_sf_stride_m + k_block_id * x_sf_stride_k,
|
||||
packed_scale,
|
||||
mask=token_mask,
|
||||
)
|
||||
else:
|
||||
sf_offset = tl.arange(0, num_groups)
|
||||
tl.store(
|
||||
x_sf
|
||||
+ token_id[:, None] * x_sf_stride_m
|
||||
+ k_block_id * x_sf_stride_k
|
||||
+ sf_offset[None, :],
|
||||
quant_scale,
|
||||
mask=token_mask[:, None],
|
||||
)
|
||||
|
||||
# DeepGEMM's SM100 shared-expert TMA loads require the activation scales
|
||||
# in an MN-major layout whose row permutation depends on the MegaMoE
|
||||
# scheduler's runtime BLOCK_M. Write that view while the packed UE8M0 scale
|
||||
# is already resident, avoiding another kernel and temporary tensor.
|
||||
if USE_UE8M0 and shared_x_sf is not None:
|
||||
shared_row = _shared_sf_row(token_id, SHARED_BLOCK_M)
|
||||
tl.store(
|
||||
shared_x_sf
|
||||
+ shared_row * shared_x_sf_stride_m
|
||||
+ k_block_id * shared_x_sf_stride_k,
|
||||
packed_scale,
|
||||
mask=token_mask,
|
||||
)
|
||||
|
||||
if k_block_id == 0:
|
||||
_store_topk(
|
||||
token_id,
|
||||
token_mask,
|
||||
topk_ids,
|
||||
topk_weights,
|
||||
is_padding,
|
||||
topk_idx_out,
|
||||
topk_weights_out,
|
||||
topk_ids_stride_m,
|
||||
topk_ids_stride_k,
|
||||
topk_weights_stride_m,
|
||||
topk_weights_stride_k,
|
||||
is_padding_stride_m,
|
||||
topk_idx_stride_m,
|
||||
topk_idx_stride_k,
|
||||
topk_weights_out_stride_m,
|
||||
topk_weights_out_stride_k,
|
||||
top_k,
|
||||
BLOCK_M,
|
||||
BLOCK_TOPK,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["num_tokens", "l2_activation_scale"])
|
||||
def _prepare_megamoe_nvfp4_inputs_kernel(
|
||||
hidden_states,
|
||||
x_out,
|
||||
x_sf,
|
||||
x_scales,
|
||||
shared_x,
|
||||
shared_x_sf,
|
||||
topk_ids,
|
||||
topk_weights,
|
||||
is_padding,
|
||||
topk_idx_out,
|
||||
topk_weights_out,
|
||||
w2_global_scale,
|
||||
l2_alpha,
|
||||
hidden_stride_m: tl.constexpr,
|
||||
hidden_stride_k: tl.constexpr,
|
||||
x_stride_m: tl.constexpr,
|
||||
x_sf_stride_m: tl.constexpr,
|
||||
shared_x_stride_m: tl.constexpr,
|
||||
shared_x_sf_stride_m: tl.constexpr,
|
||||
shared_x_sf_stride_k: tl.constexpr,
|
||||
topk_ids_stride_m: tl.constexpr,
|
||||
topk_ids_stride_k: tl.constexpr,
|
||||
topk_weights_stride_m: tl.constexpr,
|
||||
topk_weights_stride_k: tl.constexpr,
|
||||
is_padding_stride_m: tl.constexpr,
|
||||
topk_idx_stride_m: tl.constexpr,
|
||||
topk_idx_stride_k: tl.constexpr,
|
||||
topk_weights_out_stride_m: tl.constexpr,
|
||||
topk_weights_out_stride_k: tl.constexpr,
|
||||
num_tokens,
|
||||
l2_activation_scale,
|
||||
hidden_size: tl.constexpr,
|
||||
top_k: tl.constexpr,
|
||||
num_local_experts: tl.constexpr,
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_K: tl.constexpr,
|
||||
BLOCK_TOPK: tl.constexpr,
|
||||
BLOCK_E: tl.constexpr,
|
||||
SHARED_BLOCK_M: tl.constexpr,
|
||||
SHARED_MXFP8: tl.constexpr,
|
||||
) -> None:
|
||||
if l2_alpha is not None:
|
||||
experts = tl.arange(0, BLOCK_E)
|
||||
expert_mask = (experts < num_local_experts) & (tl.program_id(0) == 0)
|
||||
alpha = tl.load(w2_global_scale + experts, mask=expert_mask)
|
||||
tl.store(l2_alpha + experts, alpha * l2_activation_scale, mask=expert_mask)
|
||||
|
||||
token_id = tl.program_id(0) * BLOCK_M + tl.arange(0, BLOCK_M)
|
||||
token_mask = token_id < num_tokens
|
||||
k_offsets = tl.arange(0, BLOCK_K)
|
||||
hidden_ptrs = hidden_states + token_id[:, None] * hidden_stride_m
|
||||
|
||||
amax = tl.zeros((BLOCK_M,), tl.float32)
|
||||
for k_block_id in range(hidden_size // BLOCK_K):
|
||||
hidden = tl.load(
|
||||
hidden_ptrs + (k_block_id * BLOCK_K + k_offsets[None, :]) * hidden_stride_k,
|
||||
mask=token_mask[:, None],
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
amax = tl.maximum(amax, tl.max(tl.abs(hidden), axis=1))
|
||||
x_scale = tl.maximum(amax, 1.0e-12) * _INV_NVFP4_MAX
|
||||
tl.store(x_scales + token_id, x_scale, mask=token_mask)
|
||||
|
||||
for k_block_id in range(hidden_size // BLOCK_K):
|
||||
k_ids = k_block_id * BLOCK_K + k_offsets
|
||||
hidden = tl.load(
|
||||
hidden_ptrs + k_ids[None, :] * hidden_stride_k,
|
||||
mask=token_mask[:, None],
|
||||
other=0.0,
|
||||
)
|
||||
|
||||
# E2M1 values with one E4M3 scale per 16, relative to the token scale.
|
||||
values = tl.reshape(
|
||||
tl.div_rn(hidden.to(tl.float32), x_scale[:, None]),
|
||||
[BLOCK_M, BLOCK_K // 16, 16],
|
||||
)
|
||||
block_scale = tl.max(tl.abs(values), axis=2) * (1.0 / 6.0)
|
||||
block_scale = tl.minimum(tl.maximum(block_scale, 2.0**-9), 448.0)
|
||||
block_scale = block_scale.to(tl.float8e4nv)
|
||||
values = tl.div_rn(values, block_scale.to(tl.float32)[:, :, None])
|
||||
magnitude = tl.abs(values)
|
||||
# Round to nearest even on the grid {0, 0.5, 1, 1.5, 2, 3, 4, 6}.
|
||||
code = (
|
||||
(magnitude > 0.25).to(tl.int32)
|
||||
+ (magnitude >= 0.75).to(tl.int32)
|
||||
+ (magnitude > 1.25).to(tl.int32)
|
||||
+ (magnitude >= 1.75).to(tl.int32)
|
||||
+ (magnitude > 2.5).to(tl.int32)
|
||||
+ (magnitude >= 3.5).to(tl.int32)
|
||||
+ (magnitude > 5.0).to(tl.int32)
|
||||
)
|
||||
code = tl.where((values < 0) & (code != 0), code | 8, code)
|
||||
low, high = tl.split(tl.reshape(code, [BLOCK_M, BLOCK_K // 2, 2]))
|
||||
packed_offsets = k_block_id * (BLOCK_K // 2) + tl.arange(0, BLOCK_K // 2)
|
||||
tl.store(
|
||||
x_out + token_id[:, None] * x_stride_m + packed_offsets[None, :],
|
||||
(low | (high << 4)).to(tl.uint8),
|
||||
mask=token_mask[:, None],
|
||||
)
|
||||
sf_offsets = k_block_id * (BLOCK_K // 64) + tl.arange(0, BLOCK_K // 64)
|
||||
sf_bytes = tl.reshape(
|
||||
block_scale.to(tl.uint8, bitcast=True), [BLOCK_M, BLOCK_K // 64, 4]
|
||||
)
|
||||
tl.store(
|
||||
x_sf + token_id[:, None] * x_sf_stride_m + sf_offsets[None, :],
|
||||
_pack_bytes(sf_bytes, 4),
|
||||
mask=token_mask[:, None],
|
||||
)
|
||||
|
||||
if shared_x is not None:
|
||||
shared_ptrs = shared_x + token_id[:, None] * shared_x_stride_m
|
||||
if SHARED_MXFP8:
|
||||
fp8, _, scale_exp = _quantize_fp8_groups(
|
||||
hidden.to(tl.float32), BLOCK_M, BLOCK_K, 32, True
|
||||
)
|
||||
tl.store(shared_ptrs + k_ids[None, :], fp8, mask=token_mask[:, None])
|
||||
tl.store(
|
||||
shared_x_sf
|
||||
+ _shared_sf_row(token_id, SHARED_BLOCK_M) * shared_x_sf_stride_m
|
||||
+ k_block_id * shared_x_sf_stride_k,
|
||||
_pack_bytes(scale_exp, BLOCK_K // 32),
|
||||
mask=token_mask,
|
||||
)
|
||||
else:
|
||||
tl.store(shared_ptrs + k_ids[None, :], hidden, mask=token_mask[:, None])
|
||||
|
||||
_store_topk(
|
||||
token_id,
|
||||
token_mask,
|
||||
topk_ids,
|
||||
topk_weights,
|
||||
is_padding,
|
||||
topk_idx_out,
|
||||
topk_weights_out,
|
||||
topk_ids_stride_m,
|
||||
topk_ids_stride_k,
|
||||
topk_weights_stride_m,
|
||||
topk_weights_stride_k,
|
||||
is_padding_stride_m,
|
||||
topk_idx_stride_m,
|
||||
topk_idx_stride_k,
|
||||
topk_weights_out_stride_m,
|
||||
topk_weights_out_stride_k,
|
||||
top_k,
|
||||
BLOCK_M,
|
||||
BLOCK_TOPK,
|
||||
)
|
||||
|
||||
|
||||
def _resolve_hidden_quant(hidden_quant: QuantKey, block_k: int) -> tuple[int, bool]:
|
||||
"""Map a hidden-state ``QuantKey`` to the kernel's ``(GROUP_K, USE_UE8M0)``."""
|
||||
scale = hidden_quant.scale
|
||||
group_shape = scale.group_shape
|
||||
if (
|
||||
hidden_quant.dtype != FP8_DTYPE
|
||||
or not hidden_quant.symmetric
|
||||
or hidden_quant.scale2 is not None
|
||||
or scale.static
|
||||
or not group_shape.is_per_group()
|
||||
or block_k % group_shape.col != 0
|
||||
or scale.dtype not in (*_E8M0_SCALE_DTYPES, torch.float32)
|
||||
):
|
||||
raise ValueError(
|
||||
"MegaMoE input staging requires symmetric dynamic fp8 "
|
||||
f"per-group quantization with a group size dividing {block_k} and "
|
||||
f"E8M0 or fp32 scales, got {hidden_quant}."
|
||||
)
|
||||
return group_shape.col, scale.dtype in _E8M0_SCALE_DTYPES
|
||||
|
||||
|
||||
def prepare_megamoe_inputs(
|
||||
hidden_states: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
x: torch.Tensor,
|
||||
x_sf: torch.Tensor | None,
|
||||
topk_idx_out: torch.Tensor,
|
||||
topk_weights_out: torch.Tensor,
|
||||
is_padding: torch.Tensor | None = None,
|
||||
shared_x_sf: torch.Tensor | None = None,
|
||||
shared_block_m: int | None = None,
|
||||
hidden_quant: QuantKey | None = kMxfp8Dynamic,
|
||||
) -> None:
|
||||
"""Quantize hidden states and repack top-k routing for DeepGEMM MegaMoE.
|
||||
|
||||
Args:
|
||||
hidden_states: Input activations of shape ``[num_tokens, hidden_dim]``.
|
||||
topk_weights: Router top-k weights of shape ``[num_tokens, top_k]``.
|
||||
topk_ids: Router top-k expert ids of shape ``[num_tokens, top_k]``.
|
||||
x: Output buffer for the dispatched hidden states.
|
||||
x_sf: Output buffer for the hidden-state scale factors; ``None`` when
|
||||
``hidden_quant`` is ``None``.
|
||||
topk_idx_out: Output buffer for the repacked top-k expert ids.
|
||||
topk_weights_out: Output buffer for the repacked top-k weights.
|
||||
is_padding: Optional per-token mask; padded tokens are not routed.
|
||||
shared_x_sf: Optional output buffer for shared-expert scale factors.
|
||||
Must be given together with ``shared_block_m``.
|
||||
shared_block_m: Block M used to lay out ``shared_x_sf``.
|
||||
hidden_quant: Hidden-state quantization scheme. E8M0 scales are packed
|
||||
four per int32 into ``x_sf``; fp32 scales are stored directly.
|
||||
``None`` copies the hidden states into ``x`` unquantized.
|
||||
|
||||
"""
|
||||
num_tokens, hidden_size = hidden_states.shape
|
||||
if num_tokens == 0:
|
||||
return
|
||||
if hidden_size % 128 != 0:
|
||||
raise ValueError(
|
||||
"MegaMoE input staging requires hidden_size to be a multiple of 128."
|
||||
)
|
||||
block_k = 128
|
||||
quantize = hidden_quant is not None
|
||||
if quantize != (x_sf is not None):
|
||||
raise ValueError("MegaMoE input staging needs x_sf iff hidden_quant is set.")
|
||||
group_k, use_ue8m0 = (
|
||||
(block_k, False)
|
||||
if hidden_quant is None
|
||||
else _resolve_hidden_quant(hidden_quant, block_k)
|
||||
)
|
||||
top_k = topk_ids.shape[1]
|
||||
if topk_weights.shape != topk_ids.shape:
|
||||
raise ValueError(
|
||||
"MegaMoE input staging requires topk_weights and "
|
||||
"topk_ids to have the same shape."
|
||||
)
|
||||
if (shared_x_sf is None) != (shared_block_m is None):
|
||||
raise ValueError(
|
||||
"MegaMoE shared input staging requires both shared_x_sf and shared_block_m."
|
||||
)
|
||||
if shared_x_sf is not None and not use_ue8m0:
|
||||
raise ValueError(
|
||||
"MegaMoE shared input staging currently requires "
|
||||
"UE8M0-packed hidden scales."
|
||||
)
|
||||
if shared_x_sf is not None:
|
||||
assert shared_block_m is not None
|
||||
if shared_block_m <= 0:
|
||||
raise ValueError("MegaMoE shared_block_m must be positive.")
|
||||
expected_sf_k = hidden_size // 128
|
||||
if shared_x_sf.ndim != 2 or shared_x_sf.shape[1] != expected_sf_k:
|
||||
raise ValueError(
|
||||
"MegaMoE shared_x_sf must have shape "
|
||||
f"(*, {expected_sf_k}), got {tuple(shared_x_sf.shape)}."
|
||||
)
|
||||
aligned_block_m = triton.cdiv(shared_block_m, 128) * 128
|
||||
required_rows = triton.cdiv(num_tokens, shared_block_m) * aligned_block_m
|
||||
if shared_x_sf.shape[0] < required_rows:
|
||||
raise ValueError(
|
||||
"MegaMoE shared_x_sf has insufficient rows: requires "
|
||||
f"{required_rows}, got {shared_x_sf.shape[0]}."
|
||||
)
|
||||
|
||||
# On GB200, eight-row tiles win from 64 tokens; keep smaller batches untiled.
|
||||
block_m = 8 if num_tokens >= 64 else 1
|
||||
grid = (triton.cdiv(num_tokens, block_m), triton.cdiv(hidden_size, block_k))
|
||||
block_topk = triton.next_power_of_2(top_k)
|
||||
padding_stride_m = is_padding.stride(0) if is_padding is not None else 0
|
||||
_prepare_megamoe_inputs_kernel[grid](
|
||||
hidden_states,
|
||||
x,
|
||||
x_sf,
|
||||
shared_x_sf,
|
||||
topk_ids,
|
||||
topk_weights,
|
||||
is_padding,
|
||||
topk_idx_out,
|
||||
topk_weights_out,
|
||||
hidden_states.stride(0),
|
||||
hidden_states.stride(1),
|
||||
x.stride(0),
|
||||
x.stride(1),
|
||||
x_sf.stride(0) if x_sf is not None else 0,
|
||||
x_sf.stride(1) if x_sf is not None else 0,
|
||||
shared_x_sf.stride(0) if shared_x_sf is not None else 0,
|
||||
shared_x_sf.stride(1) if shared_x_sf is not None else 0,
|
||||
topk_ids.stride(0),
|
||||
topk_ids.stride(1),
|
||||
topk_weights.stride(0),
|
||||
topk_weights.stride(1),
|
||||
padding_stride_m,
|
||||
topk_idx_out.stride(0),
|
||||
topk_idx_out.stride(1),
|
||||
topk_weights_out.stride(0),
|
||||
topk_weights_out.stride(1),
|
||||
num_tokens,
|
||||
hidden_size,
|
||||
top_k,
|
||||
BLOCK_M=block_m,
|
||||
BLOCK_K=block_k,
|
||||
GROUP_K=group_k,
|
||||
BLOCK_TOPK=block_topk,
|
||||
SHARED_BLOCK_M=shared_block_m or 1,
|
||||
QUANTIZE=quantize,
|
||||
USE_UE8M0=use_ue8m0,
|
||||
num_warps=4,
|
||||
)
|
||||
|
||||
|
||||
def prepare_megamoe_nvfp4_inputs(
|
||||
hidden_states: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
x: torch.Tensor,
|
||||
x_sf: torch.Tensor,
|
||||
x_scales: torch.Tensor,
|
||||
topk_idx_out: torch.Tensor,
|
||||
topk_weights_out: torch.Tensor,
|
||||
is_padding: torch.Tensor | None = None,
|
||||
shared_x: torch.Tensor | None = None,
|
||||
shared_x_sf: torch.Tensor | None = None,
|
||||
shared_block_m: int | None = None,
|
||||
w2_global_scale: torch.Tensor | None = None,
|
||||
l2_alpha: torch.Tensor | None = None,
|
||||
l2_activation_scale: float = 1.0,
|
||||
) -> None:
|
||||
"""Stage DeepGEMM NVFP4 MegaMoE inputs (``kNvfp4DynamicToken``).
|
||||
|
||||
Each token is scaled by its own fp32 ``x_scales`` (row amax / 2688) and
|
||||
quantized to E2M1 with one E4M3 scale per 16 values.
|
||||
|
||||
Args:
|
||||
hidden_states: BF16 activations of shape ``[num_tokens, hidden_dim]``.
|
||||
topk_weights: Router top-k weights of shape ``[num_tokens, top_k]``.
|
||||
topk_ids: Router top-k expert ids of shape ``[num_tokens, top_k]``.
|
||||
x: Packed E2M1 output, ``[num_tokens, hidden_dim // 2]``.
|
||||
x_sf: E4M3 scales packed four per int32, ``[num_tokens, hidden_dim // 64]``.
|
||||
x_scales: Per-token fp32 dequantization scales, ``[num_tokens]``.
|
||||
topk_idx_out: Output buffer for the repacked top-k expert ids.
|
||||
topk_weights_out: Output buffer for the repacked top-k weights.
|
||||
is_padding: Optional per-token mask; padded tokens are not routed.
|
||||
shared_x: Optional shared-expert input, fp8 (MXFP8 with ``shared_x_sf``)
|
||||
or a BF16 copy of the hidden states.
|
||||
shared_x_sf: UE8M0 scales of an fp8 ``shared_x`` in the shared-expert
|
||||
TMA layout for ``shared_block_m``.
|
||||
shared_block_m: Block M used to lay out ``shared_x_sf``.
|
||||
w2_global_scale: Per-local-expert L2 weight global scales.
|
||||
l2_alpha: Output ``w2_global_scale * l2_activation_scale``, the L2
|
||||
dequantization scale of each local expert. Written even without
|
||||
local tokens, as other ranks' tokens still reach these experts.
|
||||
l2_activation_scale: Global scale the kernel quantizes L2 inputs with.
|
||||
|
||||
"""
|
||||
num_tokens, hidden_size = hidden_states.shape
|
||||
if hidden_size % 128 != 0:
|
||||
raise ValueError(
|
||||
"MegaMoE input staging requires hidden_size to be a multiple of 128."
|
||||
)
|
||||
if topk_weights.shape != topk_ids.shape:
|
||||
raise ValueError(
|
||||
"MegaMoE input staging requires topk_weights and "
|
||||
"topk_ids to have the same shape."
|
||||
)
|
||||
shared_mxfp8 = shared_x is not None and shared_x.dtype == FP8_DTYPE
|
||||
if shared_mxfp8 != (shared_x_sf is not None) or (shared_x_sf is None) != (
|
||||
shared_block_m is None
|
||||
):
|
||||
raise ValueError(
|
||||
"MegaMoE NVFP4 staging needs shared_x_sf and shared_block_m exactly "
|
||||
"for an fp8 shared_x."
|
||||
)
|
||||
if (w2_global_scale is None) != (l2_alpha is None):
|
||||
raise ValueError("MegaMoE NVFP4 staging needs w2_global_scale with l2_alpha.")
|
||||
for name, tensor in (("x", x), ("x_sf", x_sf), ("shared_x", shared_x)):
|
||||
if tensor is not None and tensor.stride(-1) != 1:
|
||||
raise ValueError(f"MegaMoE NVFP4 staging requires a row-major {name}.")
|
||||
|
||||
block_m = 8 if num_tokens >= 64 else 1
|
||||
top_k = topk_ids.shape[1]
|
||||
num_local_experts = 0 if l2_alpha is None else l2_alpha.numel()
|
||||
# One program even without tokens, to write l2_alpha.
|
||||
grid = (max(triton.cdiv(num_tokens, block_m), 1),)
|
||||
_prepare_megamoe_nvfp4_inputs_kernel[grid](
|
||||
hidden_states,
|
||||
x.view(torch.uint8),
|
||||
x_sf,
|
||||
x_scales,
|
||||
shared_x,
|
||||
shared_x_sf,
|
||||
topk_ids,
|
||||
topk_weights,
|
||||
is_padding,
|
||||
topk_idx_out,
|
||||
topk_weights_out,
|
||||
w2_global_scale,
|
||||
l2_alpha,
|
||||
hidden_states.stride(0),
|
||||
hidden_states.stride(1),
|
||||
x.stride(0),
|
||||
x_sf.stride(0),
|
||||
shared_x.stride(0) if shared_x is not None else 0,
|
||||
shared_x_sf.stride(0) if shared_x_sf is not None else 0,
|
||||
shared_x_sf.stride(1) if shared_x_sf is not None else 0,
|
||||
topk_ids.stride(0),
|
||||
topk_ids.stride(1),
|
||||
topk_weights.stride(0),
|
||||
topk_weights.stride(1),
|
||||
is_padding.stride(0) if is_padding is not None else 0,
|
||||
topk_idx_out.stride(0),
|
||||
topk_idx_out.stride(1),
|
||||
topk_weights_out.stride(0),
|
||||
topk_weights_out.stride(1),
|
||||
num_tokens,
|
||||
l2_activation_scale,
|
||||
hidden_size,
|
||||
top_k,
|
||||
num_local_experts,
|
||||
BLOCK_M=block_m,
|
||||
BLOCK_K=128,
|
||||
BLOCK_TOPK=triton.next_power_of_2(top_k),
|
||||
BLOCK_E=triton.next_power_of_2(max(num_local_experts, 1)),
|
||||
SHARED_BLOCK_M=shared_block_m or 1,
|
||||
SHARED_MXFP8=shared_mxfp8,
|
||||
num_warps=4,
|
||||
)
|
||||
@@ -79,6 +79,9 @@ class RoutedExperts(PluggableLayer):
|
||||
- Executing routed experts via quant_method.apply()
|
||||
"""
|
||||
|
||||
# Hash-routing table, owned by the gate.
|
||||
hash_indices_table: torch.Tensor | None = None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
layer_name: str,
|
||||
@@ -107,6 +110,7 @@ class RoutedExperts(PluggableLayer):
|
||||
swiglu_beta: float | None = None,
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
apply_router_weight_on_input: bool = False,
|
||||
hash_indices_table: torch.Tensor | None = None,
|
||||
):
|
||||
super().__init__()
|
||||
self.layer_name = layer_name
|
||||
@@ -152,6 +156,8 @@ class RoutedExperts(PluggableLayer):
|
||||
# Set by bind_routed_experts_capturer for monolithic kernels.
|
||||
self.routing_sink: RoutedExpertsSink | None = None
|
||||
self.apply_router_weight_on_input = apply_router_weight_on_input
|
||||
# The gate owns this Parameter: keep it out of this module's parameters.
|
||||
object.__setattr__(self, "hash_indices_table", hash_indices_table)
|
||||
# End random parameters
|
||||
self._loaded_expert_biases: set[str] = set()
|
||||
self._fused_shared_expert_quantizer = self._get_fused_shared_expert_quantizer()
|
||||
|
||||
@@ -14,7 +14,7 @@ import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.parallel_state import get_tp_group
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.forward_context import ForwardContext, get_forward_context
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
@@ -85,103 +85,74 @@ class RoutedExpertsCapturer:
|
||||
self.tp_size = vllm_config.parallel_config.tensor_parallel_size
|
||||
|
||||
def capture(self, layer_id: int, topk_ids: torch.Tensor) -> None:
|
||||
"""Capture expert routing decisions for a specific layer.
|
||||
|
||||
Under data parallelism, ``topk_ids`` may have four different batch
|
||||
layouts depending on where the DP combine happens and whether
|
||||
Expert Parallelism (EP) or Sequence Parallelism (SP) is active for the
|
||||
MoE layer:
|
||||
- ``n == total`` (naive dispatch): all DP ranks' tokens are
|
||||
concatenated before routing; we slice out this rank's span
|
||||
using the cumulative per-rank counts.
|
||||
- ``n == token_num_per_dp`` (modular-kernel path): DP combine
|
||||
happens inside ``quant_method.apply``; ``select_experts`` only
|
||||
ever sees this rank's tokens, so we take the whole tensor.
|
||||
- ``n == sum(dp_metadata.local_sizes)`` (naive DP+EP dispatch):
|
||||
sequence-parallel shards from every DP rank are gathered through
|
||||
the flattened EP group. The shard sizes include CUDA-graph / SP
|
||||
padding, so we use them to locate this DP rank's unpadded rows.
|
||||
- ``n == ceil(token_num_per_dp / tp_size)`` (SP + modular-kernel
|
||||
path): tokens were split along dim=0 across the TP group by
|
||||
``_sequence_parallel_context``
|
||||
(``moe_runner_base.py:_sequence_parallel_context``), so each
|
||||
TP rank only sees its shard. We all-gather along dim=0 to
|
||||
reconstruct this DP rank's full routing tensor. SP pads with
|
||||
ceil-div (see ``_compute_sp_num_tokens`` in
|
||||
``forward_context.py``), so the gathered tensor may contain a
|
||||
few trailing padding rows which are trimmed by the downstream
|
||||
``[:token_num_per_dp]`` slice.
|
||||
|
||||
Args:
|
||||
layer_id: The layer index.
|
||||
topk_ids: Tensor of shape (batch_size, num_routed_experts).
|
||||
|
||||
"""
|
||||
ctx = get_forward_context()
|
||||
if ctx.dp_metadata is None:
|
||||
local_topk_ids = topk_ids
|
||||
else:
|
||||
num_tokens_dp = ctx.dp_metadata.num_tokens_across_dp_cpu
|
||||
num_local_tokens = int(num_tokens_dp[self.dp_rank].item())
|
||||
total = int(num_tokens_dp.sum().item())
|
||||
n = topk_ids.shape[0]
|
||||
shard_sizes = getattr(ctx.dp_metadata, "local_sizes", None)
|
||||
gathered_size = sum(shard_sizes) if shard_sizes is not None else -1
|
||||
|
||||
if n == total:
|
||||
end = int(num_tokens_dp[: self.dp_rank + 1].sum().item())
|
||||
local_topk_ids = topk_ids[end - num_local_tokens : end]
|
||||
elif n == num_local_tokens:
|
||||
local_topk_ids = topk_ids
|
||||
elif shard_sizes is not None and n == gathered_size:
|
||||
# Naive DP+EP dispatch gathers the sequence-parallel shards in
|
||||
# flattened DP-then-TP order. ``local_sizes`` is the exact
|
||||
# all-gatherv layout, including padding. Select this DP rank's
|
||||
# contiguous shard group, then trim its trailing padding.
|
||||
num_dp_ranks = len(num_tokens_dp)
|
||||
assert len(shard_sizes) % num_dp_ranks == 0
|
||||
shards_per_dp_rank = len(shard_sizes) // num_dp_ranks
|
||||
first_shard = self.dp_rank * shards_per_dp_rank
|
||||
start = sum(shard_sizes[:first_shard])
|
||||
local_topk_ids = topk_ids[start : start + num_local_tokens]
|
||||
elif (
|
||||
self.tp_size > 1
|
||||
and n == (num_local_tokens + self.tp_size - 1) // self.tp_size
|
||||
):
|
||||
# SP + modular-kernel path. All-gather across the TP
|
||||
# group along dim=0 to reconstruct the full per-DP-rank
|
||||
# tensor; keep only the first ``token_num_per_dp`` rows
|
||||
# (trailing rows are SP ceil-div padding). The TP group
|
||||
# is always initialized on real rollout workers, and
|
||||
# every rank in the group reaches this branch in
|
||||
# lockstep (bind is per-FusedMoEFactory layer, SP is a global
|
||||
# condition), so a bare all_gather here will not
|
||||
# deadlock -- let it raise if the precondition is
|
||||
# violated rather than skip silently.
|
||||
#
|
||||
# ``topk_ids`` is already whatever the router produced
|
||||
# (typically int32/int64, both supported by NCCL); the
|
||||
# downstream buffer assignment narrows to the capture dtype.
|
||||
local_topk_ids = get_tp_group().all_gather(topk_ids, dim=0)[
|
||||
:num_local_tokens
|
||||
]
|
||||
else:
|
||||
sp_expected = (num_local_tokens + self.tp_size - 1) // self.tp_size
|
||||
raise AssertionError(
|
||||
"RoutedExpertsCapturer: unexpected topk_ids batch "
|
||||
f"dim {n} (expected {total}, {num_local_tokens}, "
|
||||
f"{gathered_size}, or {sp_expected} for "
|
||||
f"dp_rank={self.dp_rank}, tp_size={self.tp_size})"
|
||||
)
|
||||
|
||||
if not 0 <= layer_id < self.device_buffer.shape[1]:
|
||||
raise IndexError(
|
||||
f"routed-experts layer {layer_id} exceeds capture buffer "
|
||||
f"layer count {self.device_buffer.shape[1]}"
|
||||
)
|
||||
|
||||
local_topk_ids = self._restore_local_topk_ids(topk_ids, get_forward_context())
|
||||
self.device_buffer[: len(local_topk_ids), layer_id] = local_topk_ids
|
||||
|
||||
def _restore_local_topk_ids(
|
||||
self, topk_ids: torch.Tensor, ctx: ForwardContext
|
||||
) -> torch.Tensor:
|
||||
"""Restore this DP rank's token rows, trimming trailing SP padding.
|
||||
|
||||
Layout priority: cross-DP batch, local batch, gathered DP/TP shards,
|
||||
then a local TP shard. Gathered rows are DP-major/TP-minor; preserve
|
||||
this priority when layout sizes coincide. PCP is unsupported.
|
||||
|
||||
Without DP metadata, use the pre-SP batch descriptor. Unknown counts
|
||||
pass through; known counts must match a full, sharded or padded batch.
|
||||
A full local batch avoids gathering, including zero/one-token batches
|
||||
where only the publishing TP0 rank needs the real rows.
|
||||
"""
|
||||
n = topk_ids.shape[0]
|
||||
dp_metadata = ctx.dp_metadata
|
||||
if dp_metadata is None:
|
||||
if ctx.batch_descriptor is None:
|
||||
return topk_ids
|
||||
num_local_tokens = ctx.batch_descriptor.num_tokens
|
||||
else:
|
||||
num_tokens_dp = dp_metadata.num_tokens_across_dp_cpu
|
||||
num_local_tokens = int(num_tokens_dp[self.dp_rank].item())
|
||||
if n == int(num_tokens_dp.sum().item()):
|
||||
end = int(num_tokens_dp[: self.dp_rank + 1].sum().item())
|
||||
return topk_ids[end - num_local_tokens : end]
|
||||
|
||||
if n == num_local_tokens:
|
||||
return topk_ids
|
||||
|
||||
if dp_metadata is not None:
|
||||
shard_sizes = dp_metadata.local_sizes
|
||||
if shard_sizes is not None and n == sum(shard_sizes):
|
||||
num_dp_ranks = len(num_tokens_dp)
|
||||
if len(shard_sizes) % num_dp_ranks != 0:
|
||||
raise ValueError(
|
||||
"RoutedExpertsCapturer: gathered shard count "
|
||||
f"{len(shard_sizes)} is not divisible by DP rank count "
|
||||
f"{num_dp_ranks}"
|
||||
)
|
||||
shards_per_dp_rank = len(shard_sizes) // num_dp_ranks
|
||||
first_shard = self.dp_rank * shards_per_dp_rank
|
||||
start = sum(shard_sizes[:first_shard])
|
||||
return topk_ids[start : start + num_local_tokens]
|
||||
|
||||
shard_n = (num_local_tokens + self.tp_size - 1) // self.tp_size
|
||||
if n == shard_n:
|
||||
return get_tp_group().all_gather(topk_ids.contiguous(), dim=0)[
|
||||
:num_local_tokens
|
||||
]
|
||||
if dp_metadata is None and n == shard_n * self.tp_size:
|
||||
return topk_ids[:num_local_tokens]
|
||||
|
||||
raise AssertionError(
|
||||
f"RoutedExpertsCapturer: unexpected topk_ids batch dim {n} "
|
||||
f"(local_tokens={num_local_tokens}, "
|
||||
f"dp_rank={self.dp_rank}, tp_size={self.tp_size})"
|
||||
)
|
||||
|
||||
def snapshot_routing_data(self, num_tokens: int) -> torch.Tensor:
|
||||
"""Return a stable snapshot of the current routing data."""
|
||||
return self.device_buffer[:num_tokens].to(self.output_dtype)
|
||||
|
||||
+12
-2
@@ -84,7 +84,14 @@ class AiterSharedRoutedFusedMoERouter(BaseRouter):
|
||||
total_topk_weights_slice = total_topk_weights[:M]
|
||||
topk_ids_slice = total_topk_ids[:M, :topk]
|
||||
|
||||
topk_func = dispatch_topk_softmax_func(use_rocm_aiter=True)
|
||||
topk_func = dispatch_topk_softmax_func(
|
||||
use_rocm_aiter=True,
|
||||
topk_weights=total_topk_weights_slice,
|
||||
topk_indices=topk_ids_slice,
|
||||
gating_output=router_logits,
|
||||
num_shared_experts=num_fse,
|
||||
shared_expert_scoring_func="sigmoid",
|
||||
)
|
||||
topk_func(
|
||||
total_topk_weights_slice,
|
||||
topk_ids_slice,
|
||||
@@ -110,7 +117,10 @@ class AiterSharedRoutedFusedMoERouter(BaseRouter):
|
||||
)
|
||||
|
||||
topk_func = dispatch_topk_softmax_func(
|
||||
use_rocm_aiter=rocm_aiter_ops.is_fused_moe_enabled()
|
||||
use_rocm_aiter=rocm_aiter_ops.is_fused_moe_enabled(),
|
||||
topk_weights=topk_weights,
|
||||
topk_indices=topk_ids,
|
||||
gating_output=routing_logits,
|
||||
)
|
||||
topk_weights, topk_ids = topk_func(
|
||||
topk_weights,
|
||||
|
||||
@@ -61,10 +61,73 @@ def vllm_topk_sigmoid(
|
||||
return topk_weights, topk_indices
|
||||
|
||||
|
||||
# At E=512 / top-10, AITER's topk_gating is faster than the legacy launcher
|
||||
# through this many tokens; its one-row generic prefill path regresses above it.
|
||||
_AITER_TOPK_GATING_MAX_TOKENS = 4096
|
||||
|
||||
|
||||
def _aiter_topk_gating_supported(
|
||||
topk_weights: torch.Tensor,
|
||||
topk_indices: torch.Tensor,
|
||||
gating_output: torch.Tensor,
|
||||
num_shared_experts: int,
|
||||
shared_expert_scoring_func: str,
|
||||
) -> bool:
|
||||
"""Whether AITER's ``topk_gating`` can serve this softmax top-k launch.
|
||||
|
||||
It needs contiguous gating rows and at most ``_AITER_TOPK_GATING_MAX_TOKENS``
|
||||
tokens. Fused shared experts are scored with sigmoid after the routed
|
||||
top-k: 1, 2, 4 or 8 of them, with ``topk_weights`` and ``topk_indices``
|
||||
sharing a row stride and ``topk_weights`` wide enough for the shared
|
||||
columns. Shared expert ids stay as the caller prefilled them.
|
||||
"""
|
||||
if (
|
||||
not gating_output.is_contiguous()
|
||||
or gating_output.shape[0] > _AITER_TOPK_GATING_MAX_TOKENS
|
||||
):
|
||||
return False
|
||||
if num_shared_experts == 0:
|
||||
return not shared_expert_scoring_func
|
||||
return (
|
||||
shared_expert_scoring_func == "sigmoid"
|
||||
and num_shared_experts in (1, 2, 4, 8)
|
||||
and topk_weights.stride(0) == topk_indices.stride(0)
|
||||
and topk_weights.shape[-1] >= topk_indices.shape[-1] + num_shared_experts
|
||||
)
|
||||
|
||||
|
||||
def dispatch_topk_softmax_func(
|
||||
use_rocm_aiter: bool = False,
|
||||
*,
|
||||
topk_weights: torch.Tensor | None = None,
|
||||
topk_indices: torch.Tensor | None = None,
|
||||
gating_output: torch.Tensor | None = None,
|
||||
num_shared_experts: int = 0,
|
||||
shared_expert_scoring_func: str = "",
|
||||
) -> Callable[..., tuple[torch.Tensor, ...]]:
|
||||
"""Pick the softmax top-k implementation for one launch.
|
||||
|
||||
On ROCm with AITER, ``topk_gating`` is chosen on gfx942 and gfx950 when the
|
||||
launch described by the tensors and shared-expert arguments is one it supports;
|
||||
every other AITER launch uses the legacy ``topk_softmax``. The tensors are
|
||||
optional: without them the launch cannot be judged and the legacy launcher
|
||||
is used.
|
||||
"""
|
||||
if use_rocm_aiter:
|
||||
if (
|
||||
rocm_aiter_ops.is_topk_gating_enabled()
|
||||
and topk_weights is not None
|
||||
and topk_indices is not None
|
||||
and gating_output is not None
|
||||
and _aiter_topk_gating_supported(
|
||||
topk_weights,
|
||||
topk_indices,
|
||||
gating_output,
|
||||
num_shared_experts,
|
||||
shared_expert_scoring_func,
|
||||
)
|
||||
):
|
||||
return rocm_aiter_ops.topk_gating
|
||||
return rocm_aiter_ops.topk_softmax
|
||||
return vllm_topk_softmax
|
||||
|
||||
@@ -104,7 +167,10 @@ def fused_topk(
|
||||
|
||||
if scoring_func == "softmax":
|
||||
topk_func = dispatch_topk_softmax_func(
|
||||
use_rocm_aiter=rocm_aiter_ops.is_fused_moe_enabled()
|
||||
use_rocm_aiter=rocm_aiter_ops.is_fused_moe_enabled(),
|
||||
topk_weights=topk_weights,
|
||||
topk_indices=topk_ids,
|
||||
gating_output=gating_output,
|
||||
)
|
||||
topk_weights, topk_ids = topk_func(
|
||||
topk_weights, topk_ids, token_expert_indices, gating_output, renormalize
|
||||
|
||||
@@ -24,14 +24,15 @@ class GateLinear(ReplicatedLinear):
|
||||
|
||||
1. cuteDSL ll_bf16_gemm (SM90+, M<=16, bf16 in, fp32 out,
|
||||
any shape w/ K divisible by 8)
|
||||
2. fp32 specialized kernel (SM90+ or gfx950, bf16/fp32 in, fp32 out,
|
||||
M<=32, model-specific shapes)
|
||||
2. fp32 specialized kernel (bf16/fp32 in, fp32 out, M<=32; SM90+
|
||||
model-specific shapes, gfx950 any shape)
|
||||
3. cuteDSL ll_fp32w_gemm (SM90+, M<=32, bf16/fp32 in, fp32 out,
|
||||
any shape)
|
||||
4. bf16x3 CuteDSL kernel (SM100, bf16 in, fp32 weight)
|
||||
5. ROCm bf16x3 router GEMM (gfx950, bf16 in, fp32 weight, fp32 out,
|
||||
M>=2048)
|
||||
6. cuBLAS bf16×bf16→fp32 (SM90+ + bf16 weight + fp32 out_dtype)
|
||||
6. cuBLAS bf16×bf16→fp32 (SM90+ + bf16 weight + fp32 out_dtype); gfx950
|
||||
routes low M through the tier 2 kernel
|
||||
7. F.linear via ReplicatedLinear (ultimate fallback)
|
||||
|
||||
The ``out_dtype`` attribute is mutable and can be set after init
|
||||
@@ -73,16 +74,6 @@ class GateLinear(ReplicatedLinear):
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
|
||||
is_gfx950 = on_gfx950()
|
||||
is_rocm_fp32_shape = False
|
||||
if is_gfx950:
|
||||
from vllm.model_executor.layers.fused_moe.router.rocm_fp32_router_gemm import ( # noqa: E501
|
||||
ROCM_FP32_ROUTER_GEMM_SUPPORTED_SHAPES,
|
||||
)
|
||||
|
||||
is_rocm_fp32_shape = (
|
||||
input_size,
|
||||
output_size,
|
||||
) in ROCM_FP32_ROUTER_GEMM_SUPPORTED_SHAPES
|
||||
batch_invariant = envs.VLLM_BATCH_INVARIANT
|
||||
can_use_specialized_kernels = (
|
||||
current_platform.is_cuda()
|
||||
@@ -107,6 +98,7 @@ class GateLinear(ReplicatedLinear):
|
||||
return_bias=return_bias,
|
||||
)
|
||||
self.out_dtype = out_dtype
|
||||
self.is_gfx950 = is_gfx950
|
||||
|
||||
# A quantized gate exposes no plain ``weight``, so every specialized
|
||||
# tier below is disabled and forward falls back to ReplicatedLinear.
|
||||
@@ -127,7 +119,7 @@ class GateLinear(ReplicatedLinear):
|
||||
and (is_hopper or is_blackwell)
|
||||
and (input_size, output_size) in self.FP32_SUPPORTED_SHAPES
|
||||
)
|
||||
or (is_gfx950 and is_rocm_fp32_shape)
|
||||
or is_gfx950
|
||||
)
|
||||
)
|
||||
self.allow_bf16x3_router_gemm = (
|
||||
@@ -349,7 +341,10 @@ class GateLinear(ReplicatedLinear):
|
||||
|
||||
# Tier 6: cuBLAS bf16→fp32
|
||||
if self.allow_cublas_router_gemm and x.dtype == torch.bfloat16:
|
||||
output = torch.mm(x, self.weight.T, out_dtype=torch.float32)
|
||||
if self.is_gfx950:
|
||||
output = torch.ops.vllm.fp32_router_gemm_dispatch(x, self.weight, False)
|
||||
else:
|
||||
output = torch.mm(x, self.weight.T, out_dtype=torch.float32)
|
||||
return self._return(output)
|
||||
|
||||
# Tier 7: F.linear (ReplicatedLinear)
|
||||
@@ -465,17 +460,20 @@ def fp32_router_gemm_dispatch_impl(
|
||||
This must be wrapped in a custom op because our torch.compile integration
|
||||
does not support runtime dispatching on num_tokens.
|
||||
"""
|
||||
if x.shape[0] <= _FP32_ROUTER_GEMM_MAX_TOKENS:
|
||||
if current_platform.is_rocm():
|
||||
from vllm.model_executor.layers.fused_moe.router.rocm_fp32_router_gemm import ( # noqa: E501
|
||||
can_use_rocm_fp32_router_gemm,
|
||||
rocm_fp32_router_gemm,
|
||||
)
|
||||
if current_platform.is_rocm():
|
||||
from vllm.model_executor.layers.fused_moe.router.rocm_fp32_router_gemm import ( # noqa: E501
|
||||
can_use_rocm_fp32_router_gemm,
|
||||
rocm_fp32_router_gemm,
|
||||
)
|
||||
|
||||
if x.shape[0] <= _FP32_ROUTER_GEMM_MAX_TOKENS:
|
||||
x = x.contiguous()
|
||||
if can_use_rocm_fp32_router_gemm(x, weight):
|
||||
return rocm_fp32_router_gemm(x, weight)
|
||||
return torch.nn.functional.linear(x.float(), weight)
|
||||
if weight.dtype == torch.bfloat16:
|
||||
return torch.mm(x, weight.T, out_dtype=torch.float32)
|
||||
return torch.nn.functional.linear(x.float(), weight)
|
||||
if x.shape[0] <= _FP32_ROUTER_GEMM_MAX_TOKENS:
|
||||
return ops.fp32_router_gemm(x, weight)
|
||||
|
||||
if allow_bf16x3_router_gemm and x.dtype == torch.bfloat16:
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Low-token FP32 router GEMM for gfx950."""
|
||||
"""Low-token router GEMM with FP32 output for gfx950 (FP32 or BF16 weights)."""
|
||||
|
||||
import torch
|
||||
|
||||
@@ -9,15 +9,10 @@ from vllm.platforms.rocm import on_gfx950
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
_MAX_TOKENS = 32
|
||||
ROCM_FP32_ROUTER_GEMM_SUPPORTED_SHAPES = frozenset(
|
||||
{
|
||||
(3072, 256),
|
||||
(4096, 8),
|
||||
(4096, 192),
|
||||
(6144, 128),
|
||||
(6144, 256),
|
||||
}
|
||||
)
|
||||
# From this many multiply-adds on, F.linear wins on MI355.
|
||||
_MAX_MACS = 1 << 27
|
||||
# Above this num_tokens * num_experts, hipBLASLt wins for BF16 weights.
|
||||
_MAX_BF16_TOKEN_EXPERTS = 8192
|
||||
|
||||
|
||||
@triton.jit
|
||||
@@ -30,6 +25,7 @@ def _rocm_fp32_router_gemm_kernel(
|
||||
N: tl.constexpr,
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_K: tl.constexpr,
|
||||
EVEN_K: tl.constexpr,
|
||||
):
|
||||
pid = tl.program_id(0)
|
||||
pid_m = pid // N
|
||||
@@ -40,12 +36,27 @@ def _rocm_fp32_router_gemm_kernel(
|
||||
|
||||
for k_start in range(0, K, BLOCK_K):
|
||||
offsets_k_block = k_start + offsets_k
|
||||
hidden_states = tl.load(
|
||||
hidden_states_ptr + offsets_m[:, None] * K + offsets_k_block[None, :],
|
||||
mask=offsets_m[:, None] < M,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
router_weight = tl.load(router_weight_ptr + pid_n * K + offsets_k_block)
|
||||
if EVEN_K:
|
||||
hidden_states = tl.load(
|
||||
hidden_states_ptr + offsets_m[:, None] * K + offsets_k_block[None, :],
|
||||
mask=offsets_m[:, None] < M,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
router_weight = tl.load(router_weight_ptr + pid_n * K + offsets_k_block).to(
|
||||
tl.float32
|
||||
)
|
||||
else:
|
||||
k_mask = offsets_k_block < K
|
||||
hidden_states = tl.load(
|
||||
hidden_states_ptr + offsets_m[:, None] * K + offsets_k_block[None, :],
|
||||
mask=(offsets_m[:, None] < M) & k_mask[None, :],
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
router_weight = tl.load(
|
||||
router_weight_ptr + pid_n * K + offsets_k_block,
|
||||
mask=k_mask,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
partials += hidden_states * router_weight[None, :]
|
||||
|
||||
accumulator = tl.sum(partials, axis=1)
|
||||
@@ -57,28 +68,36 @@ def _rocm_fp32_router_gemm_kernel(
|
||||
|
||||
|
||||
def _launch_config(
|
||||
hidden_size: int,
|
||||
num_experts: int,
|
||||
num_tokens: int,
|
||||
num_experts: int, num_tokens: int, bf16_weight: bool
|
||||
) -> tuple[int, int, int]:
|
||||
if (hidden_size, num_experts) == (4096, 8):
|
||||
return 1, 1024, 4
|
||||
"""Return (BLOCK_M, BLOCK_K, num_warps), swept on MI355 for M<=32.
|
||||
|
||||
One program computes BLOCK_M tokens for one expert, so narrow routers keep
|
||||
BLOCK_M=1 to fill the GPU and wide routers grow BLOCK_M to reuse weights.
|
||||
"""
|
||||
if bf16_weight:
|
||||
return (1 if num_tokens <= 4 else 2), 1024, 2
|
||||
if num_experts < 64:
|
||||
return 1, 1024, 4
|
||||
wide = num_experts > 256
|
||||
if num_tokens <= 2:
|
||||
return 1, 512, 2
|
||||
if num_tokens <= 4:
|
||||
return 1, 1024, 8
|
||||
return (2, 1024, 2) if wide else (2, 2048, 4)
|
||||
if num_tokens <= 8:
|
||||
return (4, 512, 2) if wide else (4, 1024, 4)
|
||||
if num_tokens <= 16:
|
||||
block_m = 8 if (hidden_size, num_experts) == (6144, 256) else 4
|
||||
return block_m, 1024, 8
|
||||
if (hidden_size, num_experts) == (3072, 256):
|
||||
return 8, 1024, 8
|
||||
return 4, 2048, 8
|
||||
return (8, 512, 2) if wide else (4, 1024, 4)
|
||||
if num_experts >= 768:
|
||||
return 16, 256, 2
|
||||
return (8, 256, 2) if wide else (4, 512, 2)
|
||||
|
||||
|
||||
def can_use_rocm_fp32_router_gemm(
|
||||
hidden_states: torch.Tensor,
|
||||
router_weight: torch.Tensor,
|
||||
) -> bool:
|
||||
"""Return whether the tensors match the tuned gfx950 fast path."""
|
||||
"""Return whether the tensors can use the gfx950 fast path."""
|
||||
try:
|
||||
_validate_inputs(hidden_states, router_weight)
|
||||
except (RuntimeError, ValueError):
|
||||
@@ -96,26 +115,30 @@ def _validate_inputs(
|
||||
raise ValueError("hidden_states and router_weight must be 2D tensors")
|
||||
if hidden_states.dtype not in (torch.bfloat16, torch.float32):
|
||||
raise ValueError("hidden_states must have dtype bfloat16 or float32")
|
||||
if router_weight.dtype != torch.float32:
|
||||
raise ValueError("router_weight must have dtype float32")
|
||||
if router_weight.dtype not in (torch.float32, torch.bfloat16):
|
||||
raise ValueError("router_weight must have dtype float32 or bfloat16")
|
||||
if hidden_states.device.type != "cuda" or router_weight.device.type != "cuda":
|
||||
raise ValueError("hidden_states and router_weight must be GPU tensors")
|
||||
if hidden_states.device != router_weight.device:
|
||||
raise ValueError("hidden_states and router_weight must be on the same device")
|
||||
if not hidden_states.is_contiguous() or not router_weight.is_contiguous():
|
||||
raise ValueError("hidden_states and router_weight must be contiguous")
|
||||
shape = (hidden_states.shape[1], router_weight.shape[0])
|
||||
if (
|
||||
shape not in ROCM_FP32_ROUTER_GEMM_SUPPORTED_SHAPES
|
||||
or router_weight.shape[1] != shape[0]
|
||||
):
|
||||
raise ValueError(
|
||||
"supported (hidden_size, num_experts) shape pairs are "
|
||||
"(3072, 256), (4096, 8), (4096, 192), (6144, 128), "
|
||||
"and (6144, 256)"
|
||||
)
|
||||
if router_weight.shape[1] != hidden_states.shape[1]:
|
||||
raise ValueError("router_weight and hidden_states hidden sizes differ")
|
||||
if not 0 <= hidden_states.shape[0] <= _MAX_TOKENS:
|
||||
raise ValueError(f"num_tokens must be in [0, {_MAX_TOKENS}]")
|
||||
if hidden_states.shape[0] * router_weight.numel() >= _MAX_MACS:
|
||||
raise ValueError(
|
||||
f"num_tokens * num_experts * hidden_size must be < {_MAX_MACS}"
|
||||
)
|
||||
if (
|
||||
router_weight.dtype == torch.bfloat16
|
||||
and hidden_states.shape[0] * router_weight.shape[0] > _MAX_BF16_TOKEN_EXPERTS
|
||||
):
|
||||
raise ValueError(
|
||||
f"num_tokens * num_experts must be <= {_MAX_BF16_TOKEN_EXPERTS} "
|
||||
"for bfloat16 weights"
|
||||
)
|
||||
|
||||
|
||||
def rocm_fp32_router_gemm(
|
||||
@@ -131,7 +154,10 @@ def rocm_fp32_router_gemm(
|
||||
if num_tokens == 0:
|
||||
return output
|
||||
|
||||
block_m, block_k, num_warps = _launch_config(hidden_size, num_experts, num_tokens)
|
||||
block_m, block_k, num_warps = _launch_config(
|
||||
num_experts, num_tokens, router_weight.dtype == torch.bfloat16
|
||||
)
|
||||
block_k = min(block_k, triton.next_power_of_2(hidden_size))
|
||||
grid = (triton.cdiv(num_tokens, block_m) * num_experts,)
|
||||
_rocm_fp32_router_gemm_kernel[grid](
|
||||
hidden_states,
|
||||
@@ -142,6 +168,7 @@ def rocm_fp32_router_gemm(
|
||||
N=num_experts,
|
||||
BLOCK_M=block_m,
|
||||
BLOCK_K=block_k,
|
||||
EVEN_K=hidden_size % block_k == 0,
|
||||
num_warps=num_warps,
|
||||
num_stages=1,
|
||||
)
|
||||
|
||||
@@ -325,6 +325,7 @@ def _fp8_quantize(
|
||||
A_scale: torch.Tensor | None,
|
||||
per_act_token: bool,
|
||||
block_shape: list[int] | None = None,
|
||||
is_scale_transposed: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Perform fp8 quantization on the inputs. If a block_shape
|
||||
is provided, the output will be blocked.
|
||||
@@ -339,7 +340,9 @@ def _fp8_quantize(
|
||||
assert not per_act_token
|
||||
assert len(block_shape) == 2
|
||||
_, block_k = block_shape[0], block_shape[1]
|
||||
A, A_scale = per_token_group_quant_fp8(A, block_k)
|
||||
A, A_scale = per_token_group_quant_fp8(
|
||||
A, block_k, column_major_scales=is_scale_transposed
|
||||
)
|
||||
assert cdiv(A.size(-1), block_k) == A_scale.size(-1)
|
||||
|
||||
return A, A_scale
|
||||
@@ -462,6 +465,7 @@ def moe_kernel_quantize_input(
|
||||
per_act_token_quant: bool,
|
||||
block_shape: list[int] | None = None,
|
||||
is_scale_swizzled: bool = True,
|
||||
is_scale_transposed: bool = False,
|
||||
quantization_emulation: bool = False,
|
||||
mx_alignment: int = 0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
@@ -469,7 +473,9 @@ def moe_kernel_quantize_input(
|
||||
if quantization_emulation:
|
||||
return _fp8_quantize_dequantize(A, A_scale)
|
||||
else:
|
||||
return _fp8_quantize(A, A_scale, per_act_token_quant, block_shape)
|
||||
return _fp8_quantize(
|
||||
A, A_scale, per_act_token_quant, block_shape, is_scale_transposed
|
||||
)
|
||||
elif quant_dtype == torch.int8:
|
||||
if quantization_emulation:
|
||||
raise NotImplementedError(
|
||||
|
||||
@@ -31,6 +31,7 @@ from vllm.model_executor.layers.quantization.base_config import (
|
||||
)
|
||||
from vllm.model_executor.layers.utils import (
|
||||
dispatch_unquantized_gemm,
|
||||
maybe_pad_weight_avoid_cache_aliasing,
|
||||
)
|
||||
from vllm.model_executor.parameter import (
|
||||
BasevLLMParameter,
|
||||
@@ -225,6 +226,8 @@ class UnquantizedLinearMethod(LinearMethodBase):
|
||||
and weight.stride(0) != 1
|
||||
):
|
||||
layer.weight.data = weight.t().contiguous().t()
|
||||
elif current_platform.is_rocm():
|
||||
layer.weight.data = maybe_pad_weight_avoid_cache_aliasing(layer.weight.data)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
|
||||
@@ -479,6 +479,8 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
input_ids=input_ids,
|
||||
hash_indices_table=layer.hash_indices_table,
|
||||
)
|
||||
|
||||
|
||||
@@ -1132,4 +1134,6 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
input_ids=input_ids,
|
||||
hash_indices_table=layer.hash_indices_table,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,204 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Online-quantized copies of an already-loaded lm_head."""
|
||||
|
||||
from itertools import chain
|
||||
from typing import Literal
|
||||
from weakref import WeakKeyDictionary
|
||||
|
||||
import torch
|
||||
from torch.nn import Module, Parameter
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.kernels.linear import init_nvfp4_linear_kernel
|
||||
from vllm.model_executor.layers.linear import UnquantizedLinearMethod
|
||||
from vllm.model_executor.layers.quantization.online.fp8 import (
|
||||
Fp8PtpcOnlineLinearMethod,
|
||||
OnlineLinearBase,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import (
|
||||
FLOAT4_E2M1_MAX,
|
||||
ref_nvfp4_quant,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import weight_amax
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
UnquantizedEmbeddingMethod,
|
||||
)
|
||||
from vllm.model_executor.model_loader.reload.layerwise import get_layerwise_info
|
||||
from vllm.model_executor.utils import replace_parameter
|
||||
from vllm.utils.torch_utils import set_default_torch_dtype
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _quantize_nvfp4(
|
||||
weight: torch.Tensor, chunk_rows: int = 8192
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""NVFP4-quantize the rows of `weight` without the SM100+ quant kernel.
|
||||
|
||||
Returns packed E2M1 codes (low nibble first), E4M3 scales per 16 values and
|
||||
the fp32 dequantization global scale, in the checkpoint layout.
|
||||
"""
|
||||
fp8_max = torch.finfo(torch.float8_e4m3fn).max
|
||||
global_scale = (weight_amax(weight).float() / (FLOAT4_E2M1_MAX * fp8_max)).clamp(
|
||||
min=1e-12
|
||||
)
|
||||
inv_global_scale = (1.0 / global_scale).reshape(())
|
||||
# E2M1 magnitudes 0, 0.5, ..., 6 doubled to integers, mapped to their codes.
|
||||
code_of = torch.zeros(13, dtype=torch.uint8, device=weight.device)
|
||||
code_of[torch.tensor([0, 1, 2, 3, 4, 6, 8, 12])] = torch.arange(
|
||||
8, dtype=torch.uint8
|
||||
).to(weight.device)
|
||||
n, k = weight.shape
|
||||
packed = torch.empty(n, k // 2, dtype=torch.uint8, device=weight.device)
|
||||
scales = torch.empty(n, k // 16, dtype=torch.float8_e4m3fn, device=weight.device)
|
||||
for start in range(0, n, chunk_rows):
|
||||
rows = slice(start, start + chunk_rows)
|
||||
values, block_scales = ref_nvfp4_quant(weight[rows], inv_global_scale, 16)
|
||||
codes = code_of[(values.abs() * 2).long()] | ((values < 0).to(torch.uint8) << 3)
|
||||
packed[rows] = codes[:, 0::2] | (codes[:, 1::2] << 4)
|
||||
scales[rows] = block_scales.to(torch.float8_e4m3fn)
|
||||
return packed, scales, global_scale
|
||||
|
||||
|
||||
class _LMHeadNvfp4Method(OnlineLinearBase):
|
||||
"""Weight-only NVFP4 lm_head copy on the platform's W4A16 kernel;
|
||||
activations stay in the model dtype."""
|
||||
|
||||
def process_weights_after_loading(self, layer: Module) -> None:
|
||||
packed, scales, global_scale = _quantize_nvfp4(layer.weight)
|
||||
replace_parameter(layer, "weight", packed)
|
||||
layer.weight_scale = Parameter(scales, requires_grad=False)
|
||||
layer.weight_global_scale = Parameter(global_scale, requires_grad=False)
|
||||
self.kernel = init_nvfp4_linear_kernel(use_a16=True)
|
||||
self.kernel.process_weights_after_loading(layer)
|
||||
|
||||
def apply(
|
||||
self, layer: Module, x: torch.Tensor, bias: torch.Tensor | None = None
|
||||
) -> torch.Tensor:
|
||||
return self.kernel.apply_weights(layer, x, bias)
|
||||
|
||||
|
||||
_LM_HEAD_METHODS: dict[str, type[OnlineLinearBase]] = {
|
||||
"fp8": Fp8PtpcOnlineLinearMethod,
|
||||
"nvfp4": _LMHeadNvfp4Method,
|
||||
}
|
||||
|
||||
|
||||
def _quantize_copy(lm_head: ParallelLMHead, quantization: str) -> ParallelLMHead:
|
||||
if (
|
||||
not isinstance(
|
||||
lm_head.quant_method, (UnquantizedEmbeddingMethod, UnquantizedLinearMethod)
|
||||
)
|
||||
or getattr(lm_head, "bias", None) is not None
|
||||
or lm_head.weight.dtype not in (torch.float16, torch.bfloat16)
|
||||
):
|
||||
raise ValueError(
|
||||
"Quantizing an lm_head copy needs an unquantized fp16/bf16 lm_head "
|
||||
"without bias."
|
||||
)
|
||||
# Online methods take their output dtype from the default dtype, which is
|
||||
# only the model dtype while the model itself is being built.
|
||||
dtype = lm_head.weight.dtype
|
||||
with set_default_torch_dtype(dtype):
|
||||
head = ParallelLMHead(
|
||||
lm_head.num_embeddings,
|
||||
lm_head.embedding_dim,
|
||||
params_dtype=dtype,
|
||||
org_num_embeddings=lm_head.org_vocab_size,
|
||||
padding_size=lm_head.padding_size,
|
||||
prefix=lm_head.prefix,
|
||||
disable_tp=lm_head.disable_tp,
|
||||
quant_method=_LM_HEAD_METHODS[quantization](),
|
||||
)
|
||||
replace_parameter(head, "weight", lm_head.weight.data)
|
||||
head.quant_method.process_weights_after_loading(head)
|
||||
# The copy is never loaded. The online loader wrapper captures the layer,
|
||||
# so keeping it would leave each copy to the cycle collector.
|
||||
for param in head.parameters(recurse=False):
|
||||
if hasattr(param, "weight_loader"):
|
||||
del param.weight_loader
|
||||
return head
|
||||
|
||||
|
||||
# Each quantized copy maps to the head it is derived from. Weak keys, and no
|
||||
# module attribute, so the copy never owns the source as a submodule.
|
||||
_SOURCES: WeakKeyDictionary[Module, tuple[ParallelLMHead, str]] = WeakKeyDictionary()
|
||||
|
||||
|
||||
def quantized_lm_head_copy(
|
||||
lm_head: ParallelLMHead, quantization: Literal["fp8", "nvfp4"]
|
||||
) -> ParallelLMHead:
|
||||
"""A ParallelLMHead holding this rank's shard of `lm_head`, quantized.
|
||||
|
||||
"fp8" uses per-row weight and per-token activation scales on the platform's
|
||||
FP8 linear kernel; "nvfp4" is weight-only on the
|
||||
platform's W4A16 kernel. `lm_head` is left
|
||||
unchanged and no bf16 copy of its weight is made. The copy is derived state:
|
||||
`refresh_quantized_lm_heads` re-derives it after `lm_head` changes.
|
||||
"""
|
||||
head = _quantize_copy(lm_head, quantization)
|
||||
# Keep it out of layerwise reload, which would re-run
|
||||
# process_weights_after_loading on the already-quantized weight.
|
||||
get_layerwise_info(head).reset()
|
||||
_SOURCES[head] = (lm_head, quantization)
|
||||
logger.info(
|
||||
"Quantized a %s copy of the lm_head (%d x %d per rank).",
|
||||
quantization,
|
||||
head.output_size_per_partition,
|
||||
head.input_size_per_partition,
|
||||
)
|
||||
return head
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def refresh_quantized_lm_heads(model: Module) -> None:
|
||||
"""Re-derive the quantized lm_head copies in `model` from their source heads.
|
||||
|
||||
Copies in place, so CUDA graphs keep pointing at valid tensors.
|
||||
"""
|
||||
for module in model.modules():
|
||||
if module not in _SOURCES:
|
||||
continue
|
||||
fresh = _quantize_copy(*_SOURCES[module])
|
||||
for name, tensor in chain(
|
||||
fresh.named_parameters(recurse=False), fresh.named_buffers(recurse=False)
|
||||
):
|
||||
getattr(module, name).copy_(tensor)
|
||||
|
||||
|
||||
# (parent, attribute, copy) for each copy swapped out by share_source_lm_heads.
|
||||
_SWAPPED: dict[Module, list[tuple[Module, str, Module]]] = {}
|
||||
|
||||
|
||||
def share_source_lm_heads(model: Module) -> None:
|
||||
"""Replace each quantized copy in `model` with its source head.
|
||||
|
||||
`model` then loads weights like a drafter sharing the unquantized head,
|
||||
until `restore_quantized_lm_heads` swaps the copies back in.
|
||||
"""
|
||||
swapped = [
|
||||
(module, name, child)
|
||||
for module in model.modules()
|
||||
for name, child in module.named_children()
|
||||
if child in _SOURCES
|
||||
]
|
||||
for module, name, child in swapped:
|
||||
setattr(module, name, _SOURCES[child][0])
|
||||
_SWAPPED[model] = swapped
|
||||
|
||||
|
||||
def restore_quantized_lm_heads(model: Module, refresh: bool = True) -> None:
|
||||
"""Undo `share_source_lm_heads`, then re-derive the copies in `model`.
|
||||
|
||||
Args:
|
||||
model: The model passed to `share_source_lm_heads`, if any.
|
||||
refresh: Whether to re-derive the copies; False when the source heads
|
||||
may be mid-reload.
|
||||
|
||||
"""
|
||||
for module, name, head in _SWAPPED.pop(model, ()):
|
||||
setattr(module, name, head)
|
||||
if refresh:
|
||||
refresh_quantized_lm_heads(model)
|
||||
@@ -1647,8 +1647,10 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
|
||||
# If no native backend available, use emulation.
|
||||
if self.mxfp4_backend is Mxfp4MoeBackend.NONE:
|
||||
self.mxfp4_backend = Mxfp4MoeBackend.EMULATION
|
||||
self.experts_cls = None
|
||||
|
||||
self.experts_cls = backend_to_kernel_cls(self.mxfp4_backend)[0]
|
||||
if self.experts_cls is None:
|
||||
self.experts_cls = backend_to_kernel_cls(self.mxfp4_backend)[0]
|
||||
|
||||
logger.info_once(
|
||||
f"Using {self.mxfp4_backend.value} backend for {self.ocp_mx_scheme}"
|
||||
|
||||
@@ -379,6 +379,79 @@ def align_moe_weights_for_fi(
|
||||
return padded_w13, padded_w2, padded_intermediate
|
||||
|
||||
|
||||
def align_mxfp8_moe_weights_for_fi_trtllm(
|
||||
w13: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
w13_scale: torch.Tensor,
|
||||
w2_scale: torch.Tensor,
|
||||
is_act_and_mul: bool,
|
||||
min_alignment: int,
|
||||
hidden_alignment: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, int, int]:
|
||||
"""align_moe_weights_for_fi for MXFP8 weights and their UE8M0 block scales
|
||||
for the FlashInfer TRT-LLM MoE kernel, which also pads the hidden size to a
|
||||
multiple of hidden_alignment.
|
||||
|
||||
Padded weights are zero and padded scales are 127 (UE8M0 for 1.0), so the
|
||||
padded rows and columns contribute nothing. Scales may be 3D
|
||||
[E, rows, cols] or flattened [E, rows * cols]; their layout is kept.
|
||||
Returns the padded intermediate and hidden sizes.
|
||||
"""
|
||||
num_experts, hidden_size, intermediate = w2.shape
|
||||
padded_hidden = round_up(hidden_size, hidden_alignment)
|
||||
w13, w2, padded_intermediate = align_moe_weights_for_fi(
|
||||
w13, w2, is_act_and_mul, min_alignment
|
||||
)
|
||||
if padded_intermediate == intermediate and padded_hidden == hidden_size:
|
||||
return w13, w2, w13_scale, w2_scale, intermediate, hidden_size
|
||||
|
||||
if padded_hidden != hidden_size:
|
||||
logger.info_once(
|
||||
"Padding hidden size from %d to %d for MXFP8 TRTLLM MoE kernel.",
|
||||
hidden_size,
|
||||
padded_hidden,
|
||||
)
|
||||
padded_w13 = w13.new_zeros((num_experts, w13.shape[1], padded_hidden))
|
||||
padded_w13[:, :, :hidden_size] = w13
|
||||
padded_w2 = w2.new_zeros((num_experts, padded_hidden, w2.shape[2]))
|
||||
padded_w2[:, :hidden_size] = w2
|
||||
w13, w2 = padded_w13, padded_w2
|
||||
|
||||
up_mult = 2 if is_act_and_mul else 1
|
||||
w13_scale_3d = w13_scale.reshape(num_experts, up_mult * intermediate, -1)
|
||||
w2_scale_3d = w2_scale.reshape(num_experts, hidden_size, -1)
|
||||
block = hidden_size // w13_scale_3d.shape[2]
|
||||
assert padded_hidden % block == 0 and padded_intermediate % block == 0
|
||||
|
||||
# Pad each of the gate and up halves (or the single non-gated block) on its
|
||||
# own, like the weights.
|
||||
padded_w13_scale = w13_scale_3d.new_full(
|
||||
(num_experts, up_mult * padded_intermediate, padded_hidden // block), 127
|
||||
)
|
||||
for i in range(up_mult):
|
||||
dst = i * padded_intermediate
|
||||
padded_w13_scale[:, dst : dst + intermediate, : w13_scale_3d.shape[2]] = (
|
||||
w13_scale_3d[:, i * intermediate : (i + 1) * intermediate]
|
||||
)
|
||||
padded_w2_scale = w2_scale_3d.new_full(
|
||||
(num_experts, padded_hidden, padded_intermediate // block), 127
|
||||
)
|
||||
padded_w2_scale[:, :hidden_size, : w2_scale_3d.shape[2]] = w2_scale_3d
|
||||
|
||||
if w13_scale.ndim == 2:
|
||||
padded_w13_scale = padded_w13_scale.reshape(num_experts, -1)
|
||||
if w2_scale.ndim == 2:
|
||||
padded_w2_scale = padded_w2_scale.reshape(num_experts, -1)
|
||||
return (
|
||||
w13,
|
||||
w2,
|
||||
padded_w13_scale,
|
||||
padded_w2_scale,
|
||||
padded_intermediate,
|
||||
padded_hidden,
|
||||
)
|
||||
|
||||
|
||||
def _shuffle_deepseek_fp8_moe_weights(
|
||||
w13: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
@@ -486,8 +559,9 @@ def prepare_fp8_moe_layer_for_fi(
|
||||
Note that for trtllm we update the model state dict
|
||||
with the scale format needed for these kernels.
|
||||
|
||||
Note that for per-tensor, we update the layer's
|
||||
intermediate size if the weights needed padding.
|
||||
Note that for per-tensor and for MXFP8 TRT-LLM, we update the layer's
|
||||
intermediate size (and for MXFP8 TRT-LLM its hidden size) if the weights
|
||||
needed padding.
|
||||
"""
|
||||
assert hasattr(layer.moe_config, "is_act_and_mul")
|
||||
block_quant = (
|
||||
@@ -499,6 +573,21 @@ def prepare_fp8_moe_layer_for_fi(
|
||||
|
||||
# MXFP8 TRT-LLM requires W31 swap + reorder + shuffle.
|
||||
if is_mxfp8 and is_trtllm:
|
||||
# The kernel needs intermediate % 128 == 0 and hidden % 256 == 0. The
|
||||
# MoE runner pads the activations to the padded hidden size.
|
||||
w13, w2, w13_scale, w2_scale, new_intermediate, new_hidden = (
|
||||
align_mxfp8_moe_weights_for_fi_trtllm(
|
||||
w13,
|
||||
w2,
|
||||
w13_scale,
|
||||
w2_scale,
|
||||
layer.moe_config.is_act_and_mul,
|
||||
min_alignment=128,
|
||||
hidden_alignment=256,
|
||||
)
|
||||
)
|
||||
layer.moe_config.intermediate_size_per_partition = new_intermediate
|
||||
layer.moe_config.hidden_dim = new_hidden
|
||||
# FlashInfer TRT-LLM SwiGLU expects [up; gate] but vLLM stores
|
||||
# [gate; up]. Swap both weights and scales before interleaving.
|
||||
if layer.moe_config.is_act_and_mul:
|
||||
|
||||
@@ -13,6 +13,14 @@ from vllm.utils.torch_utils import direct_register_custom_op, is_torch_equal_or_
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def should_use_gfx1250_mx_scale_swizzle() -> bool:
|
||||
if not current_platform.is_rocm():
|
||||
return False
|
||||
from vllm.platforms.rocm import on_gfx1250
|
||||
|
||||
return on_gfx1250()
|
||||
|
||||
|
||||
def should_use_cdna4_mx_scale_swizzle() -> bool:
|
||||
"""Whether to use the CDNA4 swizzled scale layout for mxfp4 on gfx950.
|
||||
|
||||
@@ -27,6 +35,15 @@ def should_use_cdna4_mx_scale_swizzle() -> bool:
|
||||
return on_gfx950() and get_tensor_model_parallel_world_size() <= 2
|
||||
|
||||
|
||||
def get_aiter_mx_scale_swizzle() -> str | None:
|
||||
"""AITER `swizzle_mx_scale` arg matching the load-time scale swizzle."""
|
||||
if should_use_gfx1250_mx_scale_swizzle():
|
||||
return "GFX1250_SCALE"
|
||||
if should_use_cdna4_mx_scale_swizzle():
|
||||
return "CDNA4_SCALE"
|
||||
return None
|
||||
|
||||
|
||||
def weight_mx_scale(precision_config):
|
||||
"""mxfp4 weight scale from a triton_kernels PrecisionConfig: 3.8 exposes it
|
||||
as b_mx_scale, 3.5.1/3.6 as weight_scale.
|
||||
@@ -136,6 +153,17 @@ def _swizzle_mxfp4(quant_tensor, scale, num_warps=8):
|
||||
# transpose the tensor so that the quantization axis is on dim1
|
||||
quant_tensor = quant_tensor.transpose(-2, -1)
|
||||
scale = scale.transpose(-2, -1)
|
||||
|
||||
if should_use_gfx1250_mx_scale_swizzle():
|
||||
from aiter.ops.triton.utils.shuffle import shuffle_scale_moe
|
||||
|
||||
assert (
|
||||
scale.dim() == 3 and scale.shape[-1] % 32 == 0 and scale.shape[-2] % 4 == 0
|
||||
), f"GFX1250 scale swizzle needs (E,K_SCALE%4,N%32); got {tuple(scale.shape)}"
|
||||
scale = shuffle_scale_moe(
|
||||
scale, arch="gfx1250", preshuffle_factor=32, scale_kwidth=4
|
||||
)
|
||||
|
||||
quant_tensor = convert_layout(
|
||||
wrap_torch_tensor(quant_tensor, dtype=FP4), value_layout, **value_layout_opts
|
||||
)
|
||||
|
||||
@@ -225,6 +225,8 @@ direct_register_custom_op(
|
||||
|
||||
|
||||
def use_aiter_triton_gemm(n, m, k, dtype):
|
||||
from vllm.platforms.rocm import on_gfx1250
|
||||
|
||||
if (
|
||||
not rocm_aiter_ops.is_triton_gemm_enabled()
|
||||
# MI300's - fp8nuz=True
|
||||
@@ -233,6 +235,9 @@ def use_aiter_triton_gemm(n, m, k, dtype):
|
||||
):
|
||||
return False
|
||||
|
||||
if on_gfx1250():
|
||||
return True
|
||||
|
||||
# use hipblaslt for the larger GEMMs
|
||||
if n > 2048 and m > 512:
|
||||
return False
|
||||
@@ -285,6 +290,44 @@ def wvsplitkrc_dispatch(n: int, k: int, m: int, cu_count: int) -> tuple[int, boo
|
||||
return chunkk, fits
|
||||
|
||||
|
||||
# On gfx11 a weight row stride that is a multiple of 2048 B makes the multi-row global
|
||||
# loads of a GEMM collide on the L2/MALL channel hash. Offsetting the stride by one
|
||||
# 128 B cache line dodges the collision and keeps each row cache-line aligned.
|
||||
def cache_aliasing_pad_elems(k: int, element_size: int) -> int:
|
||||
"""Row-stride pad, in elements, that stops an ``[n, k]`` weight aliasing on gfx11,
|
||||
or 0 when the stride does not alias.
|
||||
|
||||
Args:
|
||||
k: Number of elements per weight row.
|
||||
element_size: Size of one element in bytes.
|
||||
|
||||
Returns:
|
||||
The number of elements to add to the row stride.
|
||||
|
||||
"""
|
||||
if (k * element_size) % 2048 != 0:
|
||||
return 0
|
||||
return 128 // element_size
|
||||
|
||||
|
||||
def maybe_pad_weight_avoid_cache_aliasing(weight: torch.Tensor) -> torch.Tensor:
|
||||
"""Offset a dense weight's row stride so it stops aliasing.
|
||||
|
||||
Returns a view with the original shape whose row stride is one cache line
|
||||
wider, or ``weight`` unchanged when the pad does not apply.
|
||||
"""
|
||||
from vllm.platforms.rocm import on_gfx11
|
||||
|
||||
if not (current_platform.is_rocm() and on_gfx11()):
|
||||
return weight
|
||||
if weight.dim() != 2 or not weight.is_contiguous():
|
||||
return weight
|
||||
pad = cache_aliasing_pad_elems(weight.shape[1], weight.element_size())
|
||||
if pad == 0:
|
||||
return weight
|
||||
return torch.nn.functional.pad(weight, (0, pad))[..., :-pad]
|
||||
|
||||
|
||||
def rocm_unquantized_gemm_impl(
|
||||
x: torch.Tensor, weight: torch.Tensor, bias: torch.Tensor | None = None
|
||||
) -> torch.Tensor:
|
||||
@@ -295,7 +338,7 @@ def rocm_unquantized_gemm_impl(
|
||||
|
||||
return linear_batch_invariant(x, weight, bias)
|
||||
|
||||
from vllm.platforms.rocm import on_gfx1x, on_gfx9, on_gfx950, on_gfx1250
|
||||
from vllm.platforms.rocm import on_gfx1x, on_gfx9, on_gfx11, on_gfx950
|
||||
|
||||
n = x.numel() // x.size(-1)
|
||||
m = weight.shape[0]
|
||||
@@ -328,38 +371,54 @@ def rocm_unquantized_gemm_impl(
|
||||
x_view = x.reshape(-1, x.size(-1)).contiguous()
|
||||
return ops.wvSplitKrc(x_view, weight, cu_count, bias)
|
||||
|
||||
# gfx1250's aiter gemm_a16w16 uses the gluon backend, which requires
|
||||
# K % 256 == 0 (it walks K with fixed-size descriptors and won't pad a
|
||||
# partial last tile). Some whitelisted shapes have K=2880 (e.g. gpt-oss-120b
|
||||
# hidden), so skip aiter there and fall back to the torch GEMM path below.
|
||||
if use_aiter_triton_gemm(n, m, k, x.dtype) and not (on_gfx1250() and k % 256 != 0):
|
||||
if use_aiter_triton_gemm(n, m, k, x.dtype):
|
||||
from aiter.ops.triton.gemm_a16w16 import gemm_a16w16
|
||||
|
||||
return gemm_a16w16(x, weight, bias)
|
||||
|
||||
use_skinny = (
|
||||
skinny_gemm_enabled = (
|
||||
envs.VLLM_ROCM_USE_SKINNY_GEMM
|
||||
and (on_gfx9() or on_gfx1x())
|
||||
# build (gfx9/gfx11 ISA); fall back to torch GEMM there.
|
||||
# TODO GFX1250: Include once skinny GEMM is supported on gfx1250
|
||||
and x.dtype in [torch.float16, torch.bfloat16]
|
||||
and k % 8 == 0
|
||||
and skinny_operands_compatible
|
||||
)
|
||||
|
||||
if use_skinny:
|
||||
# wvSplitK allows row strides. Perf impact is only validated on gfx11.
|
||||
weight_ok_for_wvsplitk = weight.is_contiguous() or (
|
||||
on_gfx11() and weight.stride(1) == 1
|
||||
)
|
||||
|
||||
use_wvsplitk = (
|
||||
skinny_gemm_enabled
|
||||
and weight_ok_for_wvsplitk
|
||||
and (bias is None or bias.is_contiguous())
|
||||
and (m == 1 or m > 8)
|
||||
and 0 < n <= 5
|
||||
)
|
||||
|
||||
use_llmm1 = (
|
||||
skinny_gemm_enabled
|
||||
and weight.is_contiguous()
|
||||
and bias is None
|
||||
and m % 4 == 0
|
||||
and n == 1
|
||||
and k <= 8192
|
||||
)
|
||||
|
||||
if use_wvsplitk:
|
||||
# The skinny kernels assume contiguous K elements. A shape-preserving
|
||||
# reshape can retain a transposed activation's non-contiguous strides.
|
||||
# Note: Only build that view inside the branches that consume it.
|
||||
if (m == 1 or m > 8) and 0 < n <= 5:
|
||||
x_view = x.reshape(-1, x.size(-1)).contiguous()
|
||||
cu_count = num_compute_units()
|
||||
out = ops.wvSplitK(weight, x_view, cu_count, bias)
|
||||
return out.reshape(*x.shape[:-1], weight.shape[0])
|
||||
elif m % 4 == 0 and n == 1 and k <= 8192 and bias is None:
|
||||
x_view = x.reshape(-1, x.size(-1)).contiguous()
|
||||
out = ops.LLMM1(weight, x_view, 4)
|
||||
return out.reshape(*x.shape[:-1], weight.shape[0])
|
||||
x_view = x.reshape(-1, x.size(-1)).contiguous()
|
||||
out = ops.wvSplitK(weight, x_view, cu_count, bias)
|
||||
return out.reshape(*x.shape[:-1], weight.shape[0])
|
||||
|
||||
if use_llmm1:
|
||||
x_view = x.reshape(-1, x.size(-1)).contiguous()
|
||||
out = ops.LLMM1(weight, x_view, 4)
|
||||
return out.reshape(*x.shape[:-1], weight.shape[0])
|
||||
|
||||
if rocm_aiter_ops.is_tgemm_enabled():
|
||||
from aiter.tuned_gemm import tgemm
|
||||
|
||||
@@ -607,6 +607,7 @@ class ParallelLMHead(VocabParallelEmbedding):
|
||||
prefix: str = "",
|
||||
*,
|
||||
disable_tp: bool = False,
|
||||
quant_method: QuantizeMethodBase | None = None,
|
||||
):
|
||||
self.has_bias = bias
|
||||
super().__init__(
|
||||
@@ -618,6 +619,7 @@ class ParallelLMHead(VocabParallelEmbedding):
|
||||
quant_config,
|
||||
prefix,
|
||||
disable_tp=disable_tp,
|
||||
quant_method=quant_method,
|
||||
)
|
||||
self.quant_config = quant_config
|
||||
if bias:
|
||||
|
||||
Reference in New Issue
Block a user