[MoE Refactor] Mxfp4 oracle rebased (#37128)
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
c7f98b4d0a
commit
87bd91892f
@@ -389,9 +389,9 @@ def prepare_moe_fp4_layer_for_marlin(
|
||||
|
||||
group_size = 16 if is_nvfp4 else 32
|
||||
|
||||
e = layer.num_experts
|
||||
k = layer.hidden_size
|
||||
n = layer.intermediate_size_per_partition
|
||||
e = layer.moe_config.num_experts
|
||||
k = layer.moe_config.hidden_dim
|
||||
n = layer.moe_config.intermediate_size_per_partition
|
||||
|
||||
# WORKSPACE
|
||||
device = layer.w13_weight.device
|
||||
@@ -500,6 +500,120 @@ def prepare_moe_fp4_layer_for_marlin(
|
||||
setattr(layer, name, bias)
|
||||
|
||||
|
||||
def prepare_moe_mxfp4_layer_for_marlin(
|
||||
layer: torch.nn.Module,
|
||||
w13: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
w13_scale: torch.Tensor,
|
||||
w2_scale: torch.Tensor,
|
||||
w13_bias: torch.Tensor | None,
|
||||
w2_bias: torch.Tensor | None,
|
||||
) -> tuple[
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
torch.Tensor | None,
|
||||
torch.Tensor | None,
|
||||
]:
|
||||
"""Pure-function version of prepare_moe_fp4_layer_for_marlin for MXFP4.
|
||||
|
||||
Takes weight tensors as inputs and returns transformed tensors.
|
||||
Does NOT modify the layer in-place.
|
||||
"""
|
||||
input_dtype = get_marlin_input_dtype()
|
||||
if (
|
||||
input_dtype is not None
|
||||
and input_dtype.itemsize == 1
|
||||
and input_dtype != torch.float8_e4m3fn
|
||||
):
|
||||
raise RuntimeError("MXFP4 weight + INT8 activation is not supported.")
|
||||
|
||||
group_size = 32 # MXFP4 block size
|
||||
|
||||
# Derive dimensions from actual weight shapes to handle rounded/padded
|
||||
# sizes correctly (e.g., Mxfp4MoEMethod rounds up hidden_dim).
|
||||
# w13 shape: (E, 2*N, K//2)
|
||||
e = w13.shape[0]
|
||||
n = w13.shape[1] // 2 # intermediate_size_per_partition
|
||||
k = w13.shape[2] * 2 # hidden_size
|
||||
|
||||
device = w13.device
|
||||
param_dtype = layer.params_dtype
|
||||
is_a_8bit = input_dtype is not None and input_dtype.itemsize == 1
|
||||
perm = torch.empty(0, dtype=torch.int, device=device)
|
||||
|
||||
# WEIGHT: Repack weights to marlin format
|
||||
def repack_weight(weight: torch.Tensor, name: str) -> torch.Tensor:
|
||||
tensor_list = []
|
||||
if "w13" in name:
|
||||
size_n, size_k = n * 2, k
|
||||
else:
|
||||
size_n, size_k = k, n
|
||||
|
||||
assert weight.shape == (e, size_n, size_k // 2)
|
||||
|
||||
for i in range(e):
|
||||
qweight = weight[i].view(torch.int32).T.contiguous()
|
||||
marlin_qweight = ops.gptq_marlin_repack(
|
||||
b_q_weight=qweight,
|
||||
perm=perm,
|
||||
size_k=size_k,
|
||||
size_n=size_n,
|
||||
num_bits=4,
|
||||
is_a_8bit=is_a_8bit,
|
||||
)
|
||||
tensor_list.append(marlin_qweight)
|
||||
return torch.cat([x.unsqueeze(0) for x in tensor_list], 0)
|
||||
|
||||
w13 = repack_weight(w13, "w13")
|
||||
w2 = repack_weight(w2, "w2")
|
||||
|
||||
# WEIGHT SCALES: Permute scales
|
||||
def permute_scales(scales: torch.Tensor, name: str) -> torch.Tensor:
|
||||
scales = scales.view(torch.float8_e8m0fnu)
|
||||
scales = scales.to(param_dtype)
|
||||
|
||||
tensor_list = []
|
||||
if "w13" in name:
|
||||
size_n, size_k = n * 2, k
|
||||
else:
|
||||
size_n, size_k = k, n
|
||||
|
||||
for i in range(e):
|
||||
scale = scales[i].T
|
||||
marlin_scales = marlin_permute_scales(
|
||||
s=scale,
|
||||
size_k=size_k,
|
||||
size_n=size_n,
|
||||
group_size=group_size,
|
||||
is_a_8bit=is_a_8bit,
|
||||
)
|
||||
marlin_scales = mxfp4_marlin_process_scales(
|
||||
marlin_scales, input_dtype=input_dtype
|
||||
)
|
||||
tensor_list.append(marlin_scales)
|
||||
return torch.cat([x.unsqueeze(0) for x in tensor_list], 0)
|
||||
|
||||
w13_scale = permute_scales(w13_scale, "w13")
|
||||
w2_scale = permute_scales(w2_scale, "w2")
|
||||
|
||||
# BIAS: Permute bias
|
||||
def permute_bias(bias: torch.Tensor | None) -> torch.Tensor | None:
|
||||
if bias is None:
|
||||
return None
|
||||
bias = bias.to(param_dtype)
|
||||
tensor_list = []
|
||||
for i in range(e):
|
||||
tensor_list.append(marlin_permute_bias(bias[i]))
|
||||
return torch.cat([x.unsqueeze(0) for x in tensor_list], 0)
|
||||
|
||||
w13_bias = permute_bias(w13_bias)
|
||||
w2_bias = permute_bias(w2_bias)
|
||||
|
||||
return w13, w2, w13_scale, w2_scale, w13_bias, w2_bias
|
||||
|
||||
|
||||
def rand_marlin_weight_nvfp4_like(weight, group_size, input_dtype=None):
|
||||
is_a_8bit = input_dtype is not None and input_dtype.itemsize == 1
|
||||
|
||||
|
||||
@@ -1,12 +1,10 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import triton
|
||||
from vllm.utils.import_utils import has_triton_kernels
|
||||
@@ -22,7 +20,7 @@ logger = init_logger(__name__)
|
||||
CK_MXFP4_MOE_DIM_ALIGNMENT = 256
|
||||
|
||||
|
||||
def _swizzle_mxfp4(quant_tensor, scale, num_warps):
|
||||
def _swizzle_mxfp4(quant_tensor, scale, num_warps=8):
|
||||
"""weight swizzle for mxfp4 moe, used for OAI mxfp4 kernel"""
|
||||
assert has_triton_kernels()
|
||||
import triton_kernels.matmul_ogs_details.opt_flags as opt_flags
|
||||
@@ -87,35 +85,6 @@ def _swizzle_mxfp4(quant_tensor, scale, num_warps):
|
||||
return quant_tensor, InFlexData(), scale
|
||||
|
||||
|
||||
def _can_support_mxfp4(
|
||||
use_grouped_topk: bool = False,
|
||||
topk_group: int | None = None,
|
||||
num_expert_group: int | None = None,
|
||||
expert_map: torch.Tensor | None = None,
|
||||
custom_routing_function: Callable | None = None,
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
apply_router_weight_on_input: bool = False,
|
||||
scoring_func: str = "softmax",
|
||||
activation: MoEActivation = MoEActivation.SWIGLUOAI,
|
||||
expert_load_view: torch.Tensor | None = None,
|
||||
logical_to_physical_map: torch.Tensor | None = None,
|
||||
logical_replica_count: torch.Tensor | None = None,
|
||||
):
|
||||
return not (
|
||||
use_grouped_topk
|
||||
or topk_group
|
||||
or num_expert_group
|
||||
or custom_routing_function
|
||||
or e_score_correction_bias
|
||||
or apply_router_weight_on_input
|
||||
or scoring_func != "softmax"
|
||||
or activation != MoEActivation.SWIGLUOAI
|
||||
or expert_load_view
|
||||
or logical_to_physical_map
|
||||
or logical_replica_count
|
||||
)
|
||||
|
||||
|
||||
def get_padding_alignment():
|
||||
return (
|
||||
256
|
||||
|
||||
Reference in New Issue
Block a user