[Online quantization] Add online MXFP4 quantization support (#49347)

Signed-off-by: Felix Marty <Felix.Marty@amd.com>
This commit is contained in:
fxmarty-amd
2026-08-07 12:39:46 -07:00
committed by GitHub
parent a671679e9f
commit 3518110f2f
18 changed files with 1423 additions and 35 deletions
@@ -36,6 +36,10 @@ from vllm.model_executor.layers.quantization.online.fp8 import (
from vllm.model_executor.layers.quantization.online.int8 import (
Int8OnlineMoEMethod,
)
from vllm.model_executor.layers.quantization.online.mxfp4 import (
Mxfp4OnlineLinearMethod,
Mxfp4OnlineMoEMethod,
)
from vllm.model_executor.layers.quantization.online.mxfp8 import (
Mxfp8OnlineLinearMethod,
Mxfp8OnlineMoEMethod,
@@ -49,6 +53,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8StaticChannelSym,
kFp8StaticTensorSym,
kInt8StaticChannelSym,
kMxfp4Static,
kMxfp8Dynamic,
kNvfp4Static,
)
@@ -64,6 +69,7 @@ _ONLINE_LINEAR_METHODS: dict[QuantKey, type] = {
kFp8Static128BlockSym: Fp8PerBlockOnlineLinearMethod,
kFp8StaticChannelSym: Fp8PtpcOnlineLinearMethod,
kMxfp8Dynamic: Mxfp8OnlineLinearMethod,
kMxfp4Static: Mxfp4OnlineLinearMethod,
}
_ONLINE_MOE_METHODS: dict[QuantKey, type] = {
@@ -71,6 +77,7 @@ _ONLINE_MOE_METHODS: dict[QuantKey, type] = {
kFp8Static128BlockSym: Fp8PerBlockOnlineMoEMethod,
kFp8StaticChannelSym: Fp8PtpcOnlineMoEMethod,
kMxfp8Dynamic: Mxfp8OnlineMoEMethod,
kMxfp4Static: Mxfp4OnlineMoEMethod,
kInt8StaticChannelSym: Int8OnlineMoEMethod,
kNvfp4Static: Nvfp4OnlineMoEMethod,
}
@@ -639,36 +639,6 @@ class Fp8PerBlockOnlineMoEMethod(_Fp8OnlineMoEBase):
round_up(intermediate_size_per_partition, block_size),
)
def _zero_padding(self, layer: Module) -> None:
hidden_size = layer.moe_config.hidden_dim_unpadded
intermediate_size = layer.moe_config.intermediate_size_per_partition_unpadded
w13_shard = layer.w13_weight.shape[1] // self.moe.w13_num_shards
if w13_shard > intermediate_size:
for shard in range(self.moe.w13_num_shards):
start = shard * w13_shard + intermediate_size
layer.w13_weight[:, start : (shard + 1) * w13_shard, :] = 0
if layer.w13_weight.shape[2] > hidden_size:
layer.w13_weight[:, :, hidden_size:] = 0
if layer.w2_weight.shape[1] > hidden_size:
layer.w2_weight[:, hidden_size:, :] = 0
if layer.w2_weight.shape[2] > intermediate_size:
layer.w2_weight[:, :, intermediate_size:] = 0
if getattr(layer, "w13_bias", None) is not None:
w13_bias_shard = layer.w13_bias.shape[1] // self.moe.w13_num_shards
if w13_bias_shard > intermediate_size:
for shard in range(self.moe.w13_num_shards):
start = shard * w13_bias_shard + intermediate_size
layer.w13_bias[:, start : (shard + 1) * w13_bias_shard] = 0
if (
getattr(layer, "w2_bias", None) is not None
and layer.w2_bias.shape[1] > hidden_size
):
layer.w2_bias[:, hidden_size:] = 0
def process_weights_after_loading(self, layer: Module) -> None:
if getattr(layer, "_already_called_process_weights_after_loading", False):
return
@@ -95,6 +95,36 @@ class OnlineMoEMethodBase(FusedMoEMethodBase):
initialize_online_processing(layer)
def _zero_padding(self, layer: torch.nn.Module) -> None:
hidden_size = layer.moe_config.hidden_dim_unpadded
intermediate_size = layer.moe_config.intermediate_size_per_partition_unpadded
w13_shard = layer.w13_weight.shape[1] // self.moe.w13_num_shards
if w13_shard > intermediate_size:
for shard in range(self.moe.w13_num_shards):
start = shard * w13_shard + intermediate_size
layer.w13_weight[:, start : (shard + 1) * w13_shard, :] = 0
if layer.w13_weight.shape[2] > hidden_size:
layer.w13_weight[:, :, hidden_size:] = 0
if layer.w2_weight.shape[1] > hidden_size:
layer.w2_weight[:, hidden_size:, :] = 0
if layer.w2_weight.shape[2] > intermediate_size:
layer.w2_weight[:, :, intermediate_size:] = 0
if getattr(layer, "w13_bias", None) is not None:
w13_bias_shard = layer.w13_bias.shape[1] // self.moe.w13_num_shards
if w13_bias_shard > intermediate_size:
for shard in range(self.moe.w13_num_shards):
start = shard * w13_bias_shard + intermediate_size
layer.w13_bias[:, start : (shard + 1) * w13_bias_shard] = 0
if (
getattr(layer, "w2_bias", None) is not None
and layer.w2_bias.shape[1] > hidden_size
):
layer.w2_bias[:, hidden_size:] = 0
@abstractmethod
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
pass
@@ -0,0 +1,317 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Online MXFP4 (microscaling FP4, block-32) quantization methods."""
from typing import TYPE_CHECKING
import torch
from torch.nn import Module
if TYPE_CHECKING:
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.model_executor.layers.fused_moe import (
FusedMoEParallelConfig,
FusedMoEQuantConfig,
RoutedExperts,
)
from vllm.model_executor.kernels.linear import init_mxfp4_linear_kernel
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
TRITON_BACKENDS,
Mxfp4MoeBackend,
convert_weight_to_mxfp4_moe_kernel_format,
make_mxfp4_moe_kernel,
make_mxfp4_moe_quant_config,
mxfp4_round_up_hidden_size_and_intermediate_size,
select_mxfp4_moe_backend,
)
from vllm.model_executor.layers.quantization.online.fp8 import (
_Fp8OnlineLinearBase,
)
from vllm.model_executor.layers.quantization.online.moe_base import (
OnlineMoEMethodBase,
)
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
mxfp4_quantize,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp4Dynamic
from vllm.model_executor.utils import replace_parameter
# MXFP4 block size (elements per shared e8m0 scale), same as the checkpoint
# and MoE oracle paths.
MXFP4_BLOCK_SIZE = 32
def _quantize_mxfp4_moe_weight(
weight: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Batch quantization: bf16/fp16 stacked expert weights -> MXFP4.
Returns packed FP4 weights (uint8, two values per byte) and per-block
(group-32) e8m0 scales (uint8), one pair per expert.
"""
num_experts = weight.size(0)
first_quant, first_scale = mxfp4_quantize(weight[0])
# Pre-allocate the output tensors rather than stacking.
# This is important for consistent memory layout.
w_quant = torch.empty(
(num_experts, *first_quant.shape),
dtype=first_quant.dtype,
device=weight.device,
)
w_scales = torch.empty(
(num_experts, *first_scale.shape),
dtype=first_scale.dtype,
device=weight.device,
)
w_quant[0] = first_quant
w_scales[0] = first_scale
for i in range(1, num_experts):
w_quant[i], w_scales[i] = mxfp4_quantize(weight[i])
return w_quant, w_scales
class Mxfp4OnlineLinearMethod(_Fp8OnlineLinearBase):
"""Online MXFP4 linear method.
Loads bf16/fp16 checkpoints and quantizes weights to MXFP4 (microscaling
FP4 with block-32 scales) during weight loading.
"""
activation_quant_key = kMxfp4Dynamic
def __init__(self):
super().__init__()
self.kernel = init_mxfp4_linear_kernel(
activation_quant_key=self.activation_quant_key
)
def create_weights(
self,
layer: torch.nn.Module,
input_size_per_partition: int,
output_partition_sizes: list[int],
input_size: int,
output_size: int,
params_dtype: torch.dtype,
**extra_weight_attrs,
):
if input_size_per_partition % MXFP4_BLOCK_SIZE != 0:
raise ValueError(
f"MXFP4 requires input_size_per_partition "
f"({input_size_per_partition}) to be divisible by "
f"{MXFP4_BLOCK_SIZE}."
)
super().create_weights(
layer,
input_size_per_partition,
output_partition_sizes,
input_size,
output_size,
params_dtype,
**extra_weight_attrs,
)
def process_weights_after_loading(self, layer: Module) -> None:
if getattr(layer, "_already_called_process_weights_after_loading", False):
return
weight_fp4, weight_scale = mxfp4_quantize(layer.weight.contiguous())
layer.input_scale = None
replace_parameter(layer, "weight", weight_fp4.data)
replace_parameter(layer, "weight_scale", weight_scale.data)
self.kernel.process_weights_after_loading(layer)
layer._already_called_process_weights_after_loading = True
def apply(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
return self.kernel.apply_weights(layer, x, bias)
class Mxfp4OnlineMoEMethod(OnlineMoEMethodBase):
"""MoE method for online MXFP4 (block) quantization."""
mxfp4_backend: Mxfp4MoeBackend
experts_cls: "type[mk.FusedMoEExperts] | None"
activation_quant_key = kMxfp4Dynamic
def __init__(self, *, layer: torch.nn.Module):
super().__init__(layer.moe_config)
self.weight_block_size: list[int] = [1, MXFP4_BLOCK_SIZE]
self.weight_scale_name = "weight_scale"
self.mxfp4_backend, self.experts_cls = select_mxfp4_moe_backend(
config=self.moe, activation_key=self.activation_quant_key
)
def maybe_roundup_sizes(
self,
hidden_size: int,
intermediate_size_per_partition: int,
act_dtype: torch.dtype,
moe_parallel_config: "FusedMoEParallelConfig",
) -> tuple[int, int]:
hidden_size, intermediate_size_per_partition = super().maybe_roundup_sizes(
hidden_size=hidden_size,
intermediate_size_per_partition=intermediate_size_per_partition,
act_dtype=act_dtype,
moe_parallel_config=moe_parallel_config,
)
return mxfp4_round_up_hidden_size_and_intermediate_size(
self.mxfp4_backend, hidden_size, intermediate_size_per_partition
)
def create_weights(
self,
layer: Module,
num_experts: int,
hidden_size: int,
intermediate_size_per_partition: int,
params_dtype: torch.dtype,
**extra_weight_attrs,
):
if (
hidden_size % MXFP4_BLOCK_SIZE != 0
or intermediate_size_per_partition % MXFP4_BLOCK_SIZE != 0
):
raise ValueError(
"Online MXFP4 MoE requires hidden/intermediate sizes divisible "
f"by {MXFP4_BLOCK_SIZE}."
)
super().create_weights(
layer=layer,
num_experts=num_experts,
hidden_size=hidden_size,
intermediate_size_per_partition=intermediate_size_per_partition,
params_dtype=params_dtype,
**extra_weight_attrs,
)
layer.weight_block_size = [1, MXFP4_BLOCK_SIZE]
def _setup_kernel(
self,
layer: "RoutedExperts",
w13: torch.Tensor,
w2: torch.Tensor,
w13_scale: torch.Tensor,
w2_scale: torch.Tensor,
w13_bias: torch.Tensor | None,
w2_bias: torch.Tensor | None,
) -> None:
w13, w2, w13_scale, w2_scale, w13_bias, w2_bias = (
convert_weight_to_mxfp4_moe_kernel_format(
mxfp4_backend=self.mxfp4_backend,
layer=layer,
w13_weight=w13,
w2_weight=w2,
w13_weight_scale=w13_scale,
w2_weight_scale=w2_scale,
w13_bias=w13_bias,
w2_bias=w2_bias,
)
)
# Handle weight/scale assignment based on backend type
# Copied from `quark_moe.py`.
# TODO: This should not be here, replace_parameter should handle
# triton_kernels.tensor.Tensor.
if self.mxfp4_backend in TRITON_BACKENDS or self.mxfp4_backend in (
Mxfp4MoeBackend.AITER_MXFP4_FP8,
):
# Triton-based backends: w13/w2 are triton_kernels.tensor.Tensor
# Store on layer for apply(), scales are PrecisionConfig
layer.w13_weight = w13
layer.w2_weight = w2
self.w13_precision_config = w13_scale
self.w2_precision_config = w2_scale
else:
# Standard backends: replace parameters
replace_parameter(layer, "w13_weight", w13)
replace_parameter(layer, "w2_weight", w2)
replace_parameter(layer, "w13_weight_scale", w13_scale)
replace_parameter(layer, "w2_weight_scale", w2_scale)
if w13_bias is not None and w2_bias is not None:
replace_parameter(layer, "w13_bias", w13_bias)
replace_parameter(layer, "w2_bias", w2_bias)
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
if self.moe_quant_config is not None and self.experts_cls is not None:
self.moe_kernel = make_mxfp4_moe_kernel(
moe_quant_config=self.moe_quant_config,
moe_config=self.moe,
mxfp4_backend=self.mxfp4_backend,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
layer=layer,
)
def get_fused_moe_quant_config(
self, layer: torch.nn.Module
) -> "FusedMoEQuantConfig | None":
# NOTE: unlike `QuarkOCP_MX_MoEMethod.get_fused_moe_quant_config`, this
# does not branch on `self.mxfp4_backend in TRITON_BACKENDS or
# AITER_MXFP4_FP8` to read `self.w13_precision_config`/
# `self.w2_precision_config` instead of `layer.w13_weight_scale`/
# `layer.w2_weight_scale`. For those backends,
# `convert_weight_to_mxfp4_moe_kernel_format` deletes the latter, so
# `getattr` below would raise `AttributeError`. This is currently
# unreachable because `activation_quant_key` is hardcoded to
# `kMxfp4Dynamic`
# TODO: When supporting online activation quant key override,
# fix this convert_weight_to_mxfp4_moe_kernel_format issue.
w1_scale = getattr(layer, f"w13_{self.weight_scale_name}")
w2_scale = getattr(layer, f"w2_{self.weight_scale_name}")
return make_mxfp4_moe_quant_config(
mxfp4_backend=self.mxfp4_backend,
w1_scale=w1_scale,
w2_scale=w2_scale,
w1_bias=getattr(layer, "w13_bias", None),
w2_bias=getattr(layer, "w2_bias", None),
a1_scale=layer.w13_input_scale,
a2_scale=layer.w2_input_scale,
swiglu_limit=getattr(layer, "swiglu_limit", None),
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
gemm1_beta=getattr(layer, "swiglu_beta", None),
layer=layer,
)
def process_weights_after_loading(self, layer: Module) -> None:
if getattr(layer, "_already_called_process_weights_after_loading", False):
return
self._zero_padding(layer)
if self.mxfp4_backend == Mxfp4MoeBackend.NONE:
layer._already_called_process_weights_after_loading = True
return
layer.w13_input_scale = None
layer.w2_input_scale = None
w13, w13_scale = _quantize_mxfp4_moe_weight(layer.w13_weight)
w2, w2_scale = _quantize_mxfp4_moe_weight(layer.w2_weight)
self._setup_kernel(
layer,
w13,
w2,
w13_scale,
w2_scale,
getattr(layer, "w13_bias", None),
getattr(layer, "w2_bias", None),
)
layer._already_called_process_weights_after_loading = True
@@ -6,6 +6,7 @@ import torch
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.utils.import_utils import has_triton_kernels
from vllm.utils.torch_utils import direct_register_custom_op, is_torch_equal_or_newer
@@ -184,3 +185,319 @@ except AttributeError as error:
def xpu_mxfp4_quantize(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
return torch.ops.vllm.xpu_mxfp4_quantize(x)
def permute_shape(shape: tuple[int, ...], permute_order: list[int]) -> tuple[int, ...]:
return tuple(shape[i] for i in permute_order)
@triton.jit
def _compute_quant_and_scale(src_tensor, valid_src_mask):
BLOCK_SIZE_OUT_DIM: tl.constexpr = src_tensor.shape[0]
BLOCK_SIZE_QUANT_DIM: tl.constexpr = src_tensor.shape[1]
BLOCK_SIZE_QUANT_MX_SCALE: tl.constexpr = src_tensor.shape[1] // 32
# Explicit cast to fp32 since most ops are not supported on bfloat16. We
# avoid needless conversions to and from bf16
f32_tensor = src_tensor.to(tl.float32)
abs_tensor = tl.abs(f32_tensor)
abs_tensor = tl.where(
valid_src_mask, abs_tensor, -1.0
) # Don't consider padding tensors in scale computation
abs_tensor = tl.reshape(
abs_tensor, [BLOCK_SIZE_OUT_DIM, BLOCK_SIZE_QUANT_MX_SCALE, 32]
)
max_val = tl.max(abs_tensor, axis=2, keep_dims=True)
# Round-to-nearest-even block scale (OCP MX spec), matching hardware/aiter
# MXFP4 quantizers: compute 2 ** (floor(log2(rounding(max_abs(v)))) - max_exp).
max_val = max_val.to(tl.int32, bitcast=True)
max_val = (max_val + 0x200000).to(tl.uint32, bitcast=True) & 0x7F800000
max_val = max_val.to(tl.float32, bitcast=True)
# Add epsilon to prevent log2(0) = -inf when max_val is zero (all-zero input block)
eps = tl.where(max_val == 0.0, 2 ** (-126), 0.0)
scale_e8m0_unbiased = tl.log2(max_val + eps).floor() - 2
scale_e8m0_unbiased = tl.clamp(scale_e8m0_unbiased, min=-127, max=127)
dequant_scale_rounded = tl.exp2(scale_e8m0_unbiased)
dequant_scale_exponent = dequant_scale_rounded.to(tl.uint32, bitcast=True)
dequant_scale_rounded = dequant_scale_exponent.to(tl.float32, bitcast=True)
quant_scale = tl.where(dequant_scale_rounded == 0, 0, 1.0 / dequant_scale_rounded)
f32_tensor = tl.reshape(
f32_tensor, [BLOCK_SIZE_OUT_DIM, BLOCK_SIZE_QUANT_MX_SCALE, 32]
)
quant_tensor = f32_tensor * quant_scale
# Reshape the tensors after scaling
quant_tensor = quant_tensor.reshape([BLOCK_SIZE_OUT_DIM, BLOCK_SIZE_QUANT_DIM])
# Set the invalid portions of the tensor to 0. This will ensure that any
# padding tensors are 0 in the mx format.
quant_tensor = tl.where(valid_src_mask, quant_tensor, 0)
dequant_scale_exponent = dequant_scale_exponent.reshape(
[BLOCK_SIZE_OUT_DIM, BLOCK_SIZE_QUANT_MX_SCALE]
)
# First, we simply extract the exponent part of the scales and store the result
dequant_scale_exponent = (dequant_scale_exponent >> 23).to(tl.uint8)
# Now we must convert the tensor to packed e2m1 (mxfp4).
quant_tensor = quant_tensor.to(tl.uint32, bitcast=True)
signs = quant_tensor & 0x80000000
exponents = (quant_tensor >> 23) & 0xFF
mantissas = quant_tensor & 0x7FFFFF
# 0.25 <= x < 0.75 maps to 0.5, a denormal number
E8_BIAS = 127
E2_BIAS = 1
# Move implicit bit 1 at the beginning to mantissa for denormals. Bits
# shifted out are OR-reduced into the LSB as a sticky bit, so the
# round-to-even logic below can tell an exact tie from a value that is
# merely close to one.
adjusted_exponents = tl.core.sub(E8_BIAS, exponents + 1, sanitize_overflow=False)
folded_mantissas = 0x400000 | (mantissas >> 1)
denormal_mantissas = folded_mantissas >> adjusted_exponents
denormal_mantissas |= (
(denormal_mantissas << adjusted_exponents) != folded_mantissas
).to(tl.uint32)
mantissas = tl.where(exponents < E8_BIAS, denormal_mantissas, mantissas)
# For normal numbers, we change the bias from 127 to 1, and for subnormals,
# we keep exponent as 0.
exponents = tl.maximum(exponents, E8_BIAS - E2_BIAS) - (E8_BIAS - E2_BIAS)
# Combine sign, exponent, and mantissa, rounding to nearest with ties
# broken to even (OCP MX spec, matching hardware/aiter MXFP4
# quantizers) via the standard "add a rounding bias, then truncate"
# trick: the bias's LSB is the kept mantissa bit, so an exact tie
# rounds to whichever candidate is even, while any lower (sticky) bit
# being set always rounds up.
rounding_bias = ((mantissas >> 22) & 1) + 0x1FFFFF
rounded_mantissa = (mantissas + rounding_bias) >> 22 # in {0, 1, 2}
e2m1_tmp = tl.minimum((exponents << 1) + rounded_mantissa, 0x7)
e2m1_value = ((signs >> 28) | e2m1_tmp).to(tl.uint8)
e2m1_value = tl.reshape(
e2m1_value, [BLOCK_SIZE_OUT_DIM, BLOCK_SIZE_QUANT_DIM // 2, 2]
)
evens, odds = tl.split(e2m1_value)
out_tensor = evens | (odds << 4)
return out_tensor, dequant_scale_exponent
@triton.jit
def _downcast_to_mxfp(
mx_tensor_ptr,
stride_mxt_outer,
stride_mxt_quant: tl.constexpr,
mx_scale_ptr,
stride_mx_scale_outer,
stride_mx_scale_quant,
src_ptr,
stride_src_outer,
stride_src_quant,
outer_dim,
quant_dim,
BLOCK_SIZE_OUT_DIM: tl.constexpr,
BLOCK_SIZE_QUANT_DIM: tl.constexpr,
):
tl.static_assert(
stride_mxt_quant == 1, f"Output stride, {stride_mxt_quant=} must be 1."
)
tl.static_assert(
BLOCK_SIZE_QUANT_DIM % 32 == 0,
f"{BLOCK_SIZE_QUANT_DIM=} must be a multiple of 32",
)
# uint8 signifies two fp4 e2m1 values packed into a single byte
tl.static_assert(
mx_tensor_ptr.dtype.element_ty == tl.uint8,
f"Invalid {mx_tensor_ptr.dtype.element_ty=}. Must be uint8.",
)
src_dtype: tl.constexpr = src_ptr.dtype.element_ty
tl.static_assert(
mx_scale_ptr.dtype.element_ty == tl.uint8,
f"{mx_scale_ptr.dtype.element_ty=} must be uint8",
)
tl.static_assert(
(src_dtype == tl.bfloat16) or (src_dtype == tl.float16),
f"{src_dtype=} must be bfloat16 or float16",
)
outer_block = tl.program_id(0).to(tl.int64)
quant_block = tl.program_id(1).to(tl.int64)
BLOCK_SIZE_QUANT_MX_SCALE: tl.constexpr = BLOCK_SIZE_QUANT_DIM // 32
BLOCK_SIZE_QUANT_MX_TENSOR: tl.constexpr = BLOCK_SIZE_QUANT_DIM // 2
start_src_quant = quant_block * BLOCK_SIZE_QUANT_DIM
start_mx_scale_quant = quant_block * BLOCK_SIZE_QUANT_MX_SCALE
start_mx_quant = quant_block * BLOCK_SIZE_QUANT_MX_TENSOR
start_out = outer_block * BLOCK_SIZE_OUT_DIM
src_ptr += start_src_quant * stride_src_quant + start_out * stride_src_outer
mx_scale_ptr += (
start_mx_scale_quant * stride_mx_scale_quant + start_out * stride_mx_scale_outer
)
mx_tensor_ptr += start_mx_quant * stride_mxt_quant + start_out * stride_mxt_outer
offs_src_quant = tl.arange(0, BLOCK_SIZE_QUANT_DIM)[None, :].to(tl.int64)
offs_mxt_quant = tl.arange(0, BLOCK_SIZE_QUANT_MX_TENSOR)[None, :].to(tl.int64)
offs_scale_quant = tl.arange(0, BLOCK_SIZE_QUANT_MX_SCALE)[None, :].to(tl.int64)
offs_outer = tl.arange(0, BLOCK_SIZE_OUT_DIM)[:, None].to(tl.int64)
mask_src_quant = start_src_quant + offs_src_quant < quant_dim
mask_n = start_out + offs_outer < outer_dim
full_mask_src = mask_src_quant and mask_n
mask_mxt_quant = start_mx_quant + offs_mxt_quant < tl.cdiv(quant_dim, 2)
full_mask_mxt = mask_mxt_quant and mask_n
scale_mask_k = start_mx_scale_quant + offs_scale_quant < tl.cdiv(quant_dim, 32)
full_scale_mask = scale_mask_k and mask_n
src_tensor_offsets = (
offs_src_quant * stride_src_quant + offs_outer * stride_src_outer
)
mx_scale_offsets = (
offs_scale_quant * stride_mx_scale_quant + offs_outer * stride_mx_scale_outer
)
mx_tensor_offsets = (
offs_mxt_quant * stride_mxt_quant + offs_outer * stride_mxt_outer
)
src_tensor = tl.load(src_ptr + src_tensor_offsets, mask=full_mask_src)
out_tensor, scale_tensor = _compute_quant_and_scale(src_tensor, full_mask_src)
tl.store(mx_scale_ptr + mx_scale_offsets, scale_tensor, mask=full_scale_mask)
tl.store(mx_tensor_ptr + mx_tensor_offsets, out_tensor, mask=full_mask_mxt)
def downcast_to_mxfp(
src_tensor: torch.Tensor,
axis: int,
out_quant_tensor: torch.Tensor | None = None,
out_scale: torch.Tensor | None = None,
BLOCK_OUT_DIM: int = 128,
BLOCK_QUANT_DIM: int = 32,
):
"""
Convert the src weights to MXFP4. The src weight is quantized along the
axis dimension into packed e2m1 values (torch.uint8, two values per
byte), so the size of that dimension in the output is half of the
logical (unpacked) size.
"""
out_quant_type = torch.uint8
ndim = src_tensor.ndim
assert -ndim <= axis < ndim, f"Invalid axis {axis=}"
axis = axis if axis >= 0 else axis + ndim
L = src_tensor.shape[axis]
# We make this assertion since we can't track if the "real" shape was odd,
# and we padded it to be even.
# We want to maintain the property dequant(quant(x)).shape == x.shape
assert L % 2 == 0, f"axis dim must be divisible by 2 for e2m1. Got {L}"
device = src_tensor.device
packed_quant_dim = triton.cdiv(L, 2)
out_scale_dim = triton.cdiv(L, 32)
# Move the quantization axis to the end for the kernel, then permute back.
permute_order = list(range(ndim))
permute_order[axis], permute_order[-1] = permute_order[-1], permute_order[axis]
prmted_quant_tensor_shape = permute_shape(src_tensor.shape, permute_order)[:-1] + (
packed_quant_dim,
)
prmted_scale_shape = permute_shape(src_tensor.shape, permute_order)[:-1] + (
out_scale_dim,
)
prmted_src_tensor = src_tensor.permute(permute_order)
if out_quant_tensor is None:
out_quant_tensor = torch.empty(
prmted_quant_tensor_shape, dtype=out_quant_type, device=device
)
else:
expected_shape = (
src_tensor.shape[:axis] + (packed_quant_dim,) + src_tensor.shape[axis + 1 :]
)
assert out_quant_tensor.shape == expected_shape, (
f"{out_quant_tensor.shape=} != {expected_shape=}"
)
assert out_quant_tensor.dtype == out_quant_type, (
f"{out_quant_tensor.dtype=} != {out_quant_type=}"
)
assert out_quant_tensor.stride(axis) == 1, (
f"{out_quant_tensor.stride(axis)=} != 1"
)
# We expect the axis dimension to be last, so permute the tensor
out_quant_tensor = out_quant_tensor.permute(permute_order)
if out_scale is None:
out_scale = torch.empty(prmted_scale_shape, dtype=torch.uint8, device=device)
else:
expected_scale_shape = permute_shape(prmted_scale_shape, permute_order)
assert out_scale.shape == expected_scale_shape, (
f"{out_scale.shape=} {expected_scale_shape=}"
)
assert out_scale.dtype == torch.uint8, f"{out_scale.dtype=} != torch.uint8"
out_scale = out_scale.permute(permute_order)
# Flatten input tensor for kernel. This will typically make a copy
reshaped_src_tensor = prmted_src_tensor.reshape(-1, L)
blocks_quant_dim = triton.cdiv(reshaped_src_tensor.shape[-1], BLOCK_QUANT_DIM)
blocks_out_dim = triton.cdiv(reshaped_src_tensor.shape[0], BLOCK_OUT_DIM)
# Flatten the output tensors for the kernel, this should be a view always
kernel_quant_tensor = out_quant_tensor.reshape(-1, packed_quant_dim)
kernel_scale = out_scale.reshape(-1, out_scale_dim)
assert kernel_quant_tensor.data_ptr() == out_quant_tensor.data_ptr()
assert kernel_scale.data_ptr() == out_scale.data_ptr()
_downcast_to_mxfp[(blocks_out_dim, blocks_quant_dim)](
kernel_quant_tensor,
*kernel_quant_tensor.stride(),
kernel_scale,
*kernel_scale.stride(),
reshaped_src_tensor,
*reshaped_src_tensor.stride(),
*reshaped_src_tensor.shape,
BLOCK_OUT_DIM,
BLOCK_QUANT_DIM,
num_warps=8,
)
out_quant_tensor = out_quant_tensor.permute(permute_order)
out_scale = out_scale.permute(permute_order).contiguous()
return out_quant_tensor, out_scale, permute_shape(prmted_scale_shape, permute_order)
def mxfp4_quantize(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
"""Quantize a bf16/fp16 tensor to MXFP4 along its last dimension.
Dispatches to the fastest backend available on the current platform:
- the native XPU custom op on XPU,
- aiter on ROCm when `aiter` is installed,
- and the portable Triton kernel (``downcast_to_mxfp``) otherwise.
Returns packed FP4 values (uint8, two values per byte) and per-block
(group-32) e8m0 scales (uint8).
"""
if current_platform.is_xpu():
return xpu_mxfp4_quantize(x)
from vllm._aiter_ops import is_aiter_found_and_supported
if is_aiter_found_and_supported() and x.dtype == torch.bfloat16:
from vllm.model_executor.layers.quantization.quark.utils import (
quark_quantize_weight_to_mxfp4,
)
return quark_quantize_weight_to_mxfp4(x)
quant_tensor, scale, _ = downcast_to_mxfp(x, axis=-1)
return quant_tensor, scale